non-greedy повторители

Програмиране с Go

Курс във Факултета по Математика и Информатика към СУ

Решение на Concurrent Crawling от Йосиф Цветков

Обратно към всички решения

Към профила на Йосиф Цветков

Резултати

  • 8 точки от тестове
  • 0 бонус точки
  • 8 точки общо
  • 9 успешни тест(а)
  • 2 неуспешни тест(а)

Код

package main
import "time"
import "net/http"
import "io/ioutil"
type CallError struct {
message string
}
func (ce *CallError) Error() string {
return ce.message
}
func SeekAndDestroy(callback func(string) bool, chunkedUrlsToCheck <-chan []string, workersCount int) (str string, err error) {
if workersCount < 1 {
return "", &CallError{message: "Invalid value for \"workersCount\" !!!"}
} else if chunkedUrlsToCheck == nil {
return "", &CallError{message: "Uninitialised channel !!!"}
}
result := make(chan string)
kill := make(chan struct{}, 1)
defer close(result)
defer close(kill)
go startRoutines(result, kill, chunkedUrlsToCheck, workersCount, callback)
select {
case res := <-result:
kill <- struct{}{}
if res == "nil" {
return "", &CallError{message: "Nothing found !!!"}
}
return res, nil
case _ = <-time.After(15 * time.Second):
kill <- struct{}{}
return "", &CallError{message: "Nothing found !!!"}
}
}
func startRoutines(result chan string, seppuku chan struct{}, chunkedUrlsToCheck <-chan []string, workersCount int, callback func(string) bool) {
poolOfWorkers := make(chan struct{}, workersCount)
for i := 0; i < workersCount; i++ {
poolOfWorkers <- struct{}{}
}
i := 0
killSliceRoutine := make(map[int]chan struct{})
for {
select {
case _ = <-seppuku:
for _, killer := range killSliceRoutine {
killer <- struct{}{}
close(killer)
}
return
case urlSlice := <-chunkedUrlsToCheck:
if len(urlSlice) == 0 {
result <- "nil"
return
}
killSliceRoutine[i] = make(chan struct{})
go func(death chan struct{}, urlSlice []string) {
for _, url := range urlSlice {
select {
case _ = <-poolOfWorkers:
stringMatch := make(chan string, 2)
stringNotMatch := make(chan struct{})
go func(url string, stringMatch chan string, stringNotMatch chan struct{}) {
response, err := http.Get(url)
if err != nil {
stringNotMatch <- struct{}{}
} else {
defer response.Body.Close()
contents, err2 := ioutil.ReadAll(response.Body)
if err2 != nil {
stringNotMatch <- struct{}{}
return
}
stringMatch <- string(contents)
stringMatch <- url
}
}(url, stringMatch, stringNotMatch)
go func(stringMatch chan string, stringNotMatch chan struct{}) {
select {
case _ = <-time.After(3 * time.Second):
poolOfWorkers <- struct{}{}
case httpContent := <-stringMatch:
if callback(httpContent) {
successURL := <-stringMatch
result <- successURL
} else {
poolOfWorkers <- struct{}{}
}
case _ = <-stringNotMatch:
poolOfWorkers <- struct{}{}
}
}(stringMatch, stringNotMatch)
case _ = <-death:
return
}
}
}(killSliceRoutine[i], urlSlice)
i++
}
}
}

Лог от изпълнението

[/tmp/go-build200095534/_/tmp/d20150111-16649-goe0vk/_test/d20150111-16649-goe0vk.test -test.run=TestWithNegativeWorkersCount -test.timeout=120s]
PASS
ok  	_/tmp/d20150111-16649-goe0vk	0.008s
[/tmp/go-build451788621/_/tmp/d20150111-16649-goe0vk/_test/d20150111-16649-goe0vk.test -test.run=TestWithZeroWorkersCount -test.timeout=120s]
PASS
ok  	_/tmp/d20150111-16649-goe0vk	0.007s
[/tmp/go-build471734946/_/tmp/d20150111-16649-goe0vk/_test/d20150111-16649-goe0vk.test -test.run=TestWithInvalidCallback -test.timeout=120s]
--- FAIL: TestWithInvalidCallback-2 (1.00 seconds)
	solution_test.go:43: Test exceeded allowed time of 1 seconds: parameter errors should be immediately returned (callback is nil)
FAIL
exit status 1
FAIL	_/tmp/d20150111-16649-goe0vk	1.007s
[/tmp/go-build907540865/_/tmp/d20150111-16649-goe0vk/_test/d20150111-16649-goe0vk.test -test.run=TestWithNilChannel -test.timeout=120s]
PASS
ok  	_/tmp/d20150111-16649-goe0vk	0.006s
[/tmp/go-build445600486/_/tmp/d20150111-16649-goe0vk/_test/d20150111-16649-goe0vk.test -test.run=TestWithClosedChannelWhenStarting -test.timeout=120s]
PASS
ok  	_/tmp/d20150111-16649-goe0vk	0.006s
[/tmp/go-build974989735/_/tmp/d20150111-16649-goe0vk/_test/d20150111-16649-goe0vk.test -test.run=TestWithClosedChannelMidway -test.timeout=120s]
PASS
ok  	_/tmp/d20150111-16649-goe0vk	5.005s
[/tmp/go-build663586387/_/tmp/d20150111-16649-goe0vk/_test/d20150111-16649-goe0vk.test -test.run=TestWhetherGlobalTimeoutIsHandled -test.timeout=120s]
PASS
ok  	_/tmp/d20150111-16649-goe0vk	15.007s
[/tmp/go-build734235799/_/tmp/d20150111-16649-goe0vk/_test/d20150111-16649-goe0vk.test -test.run=TestWithLoremIpsum -test.timeout=120s]
PASS
ok  	_/tmp/d20150111-16649-goe0vk	2.009s
[/tmp/go-build736197663/_/tmp/d20150111-16649-goe0vk/_test/d20150111-16649-goe0vk.test -test.run=TestIfTimeoutAndErrorCodesAreHonoured -test.timeout=120s]
--- FAIL: TestIfTimeoutAndErrorCodesAreHonoured-2 (0.00 seconds)
	solution_test.go:267: Function returned 'http://127.0.0.2:52111/page_with_error_code' when it should have returned 'http://127.0.0.2:52111/correct_page'
FAIL
exit status 1
FAIL	_/tmp/d20150111-16649-goe0vk	0.006s
[/tmp/go-build211722919/_/tmp/d20150111-16649-goe0vk/_test/d20150111-16649-goe0vk.test -test.run=TestRaceCondition -test.timeout=120s]
PASS
ok  	_/tmp/d20150111-16649-goe0vk	1.010s
[/tmp/go-build243785528/_/tmp/d20150111-16649-goe0vk/_test/d20150111-16649-goe0vk.test -test.run=TestCloseChannelBeforeFinish -test.timeout=120s]
PASS
ok  	_/tmp/d20150111-16649-goe0vk	1.006s

История (4 версии и 3 коментара)

Йосиф обнови решението на 05.12.2014 01:06 (преди над 3 години)

+package main
+
+import "time"
+
+type CallError struct {
+ message string
+}
+
+func (ce *CallError) Error() string {
+ return ce.message
+}
+
+func SeekAndDestroy(callback func(string) bool, chunkedUrlsToCheck <-chan []string, workersCount int) (str string, err error) {
+ if workersCount < 1 {
+ return "", &CallError{message: "Invalid value for \"workersCount\" !!!"}
+ } else if chunkedUrlsToCheck == nil {
+ return "", &CallError{message: "Uninitialised channel !!!"}
+ }
+
+ result := make(chan string)
+ kill := make(chan struct{})
+
+ go startRoutines(result, kill, chunkedUrlsToCheck, workersCount, callback)
+
+ select {
+ case res := <-result:
+ kill <- struct{}{}
+ return res, nil
+ case _ = <-time.After(15 * time.Second):
+ kill <- struct{}{}
+ return "", &CallError{message: "Nothing found !!!"}
+ }
+}
+
+func startRoutines(result chan string, ownDeath chan struct{}, chunkedUrlsToCheck <-chan []string, workersCount int, callback func(string) bool) {
+ pool := make(chan struct{}, workersCount)
+ for i := 0; i < workersCount; i++ {
+ pool <- struct{}{}
+ }
+
+ i := 0
+ killSliceRoutine := make(map[int]chan struct{})
+
+ for {
+ select {
+ case _ = <-ownDeath:
+ for _, killMachine := range killSliceRoutine {
+ killMachine <- struct{}{}
+ }
+ return
+ case urlSlice := <-chunkedUrlsToCheck:
+ killSliceRoutine[i] = make(chan struct{})
+ go func(death chan struct{}, urlSlice []string) {
+ chans := make([]chan string, len(urlSlice))
+ for _, url := range urlSlice {
+ select {
+ case _ = <-pool:
+ newChan := make(chan string)
+ chans = append(chans, newChan)
+
+ //worker-routine
+ go func(url string, newChan chan string) {
+ if callback(url) {
+ newChan <- url
+ }
+ }(url, newChan)
+
+ go func(newChan chan string) {
+ select {
+ case _ = <-time.After(3 * time.Second):
+ pool <- struct{}{}
+ case match := <-newChan:
+ result <- match
+ }
+ }(newChan)
+
+ case _ = <-death:
+ return
+ }
+ }
+ }(killSliceRoutine[i], urlSlice)
+ i++
+ }
+ }
+}

Йосиф обнови решението на 09.12.2014 14:26 (преди над 3 години)

package main
import "time"
type CallError struct {
message string
}
func (ce *CallError) Error() string {
return ce.message
}
func SeekAndDestroy(callback func(string) bool, chunkedUrlsToCheck <-chan []string, workersCount int) (str string, err error) {
if workersCount < 1 {
return "", &CallError{message: "Invalid value for \"workersCount\" !!!"}
} else if chunkedUrlsToCheck == nil {
return "", &CallError{message: "Uninitialised channel !!!"}
}
result := make(chan string)
kill := make(chan struct{})
go startRoutines(result, kill, chunkedUrlsToCheck, workersCount, callback)
select {
case res := <-result:
kill <- struct{}{}
return res, nil
case _ = <-time.After(15 * time.Second):
kill <- struct{}{}
return "", &CallError{message: "Nothing found !!!"}
}
}
func startRoutines(result chan string, ownDeath chan struct{}, chunkedUrlsToCheck <-chan []string, workersCount int, callback func(string) bool) {
- pool := make(chan struct{}, workersCount)
+ poolOfWorkers := make(chan struct{}, workersCount)
for i := 0; i < workersCount; i++ {
- pool <- struct{}{}
+ poolOfWorkers <- struct{}{}
}
i := 0
killSliceRoutine := make(map[int]chan struct{})
for {
select {
case _ = <-ownDeath:
- for _, killMachine := range killSliceRoutine {
- killMachine <- struct{}{}
+ for _, killer := range killSliceRoutine {
+ killer <- struct{}{}
}
return
case urlSlice := <-chunkedUrlsToCheck:
killSliceRoutine[i] = make(chan struct{})
go func(death chan struct{}, urlSlice []string) {
- chans := make([]chan string, len(urlSlice))
for _, url := range urlSlice {
select {
- case _ = <-pool:
- newChan := make(chan string)
- chans = append(chans, newChan)
-
- //worker-routine
- go func(url string, newChan chan string) {
+ case _ = <-poolOfWorkers:
+ stringMatch := make(chan string)
+ stringNotMatch := make(chan struct{})
+ go func(url string, stringMatch chan string, stringNotMatch chan struct{}) {
if callback(url) {
- newChan <- url
+ stringMatch <- url
+ } else {
+ stringNotMatch <- struct{}{}
}
- }(url, newChan)
-
- go func(newChan chan string) {
+ }(url, stringMatch, stringNotMatch)
+ go func(stringMatch chan string, stringNotMatch chan struct{}) {
select {
case _ = <-time.After(3 * time.Second):
- pool <- struct{}{}
- case match := <-newChan:
+ poolOfWorkers <- struct{}{}
+ case match := <-stringMatch:
result <- match
+ case _ = <-stringNotMatch:
+ poolOfWorkers <- struct{}{}
}
- }(newChan)
-
+ }(stringMatch, stringNotMatch)
case _ = <-death:
return
}
}
}(killSliceRoutine[i], urlSlice)
i++
}
}
}

Йосиф обнови решението на 10.12.2014 22:43 (преди над 3 години)

package main
import "time"
+import "net/http"
+import "io/ioutil"
type CallError struct {
message string
}
func (ce *CallError) Error() string {
return ce.message
}
func SeekAndDestroy(callback func(string) bool, chunkedUrlsToCheck <-chan []string, workersCount int) (str string, err error) {
if workersCount < 1 {
return "", &CallError{message: "Invalid value for \"workersCount\" !!!"}
} else if chunkedUrlsToCheck == nil {
return "", &CallError{message: "Uninitialised channel !!!"}
}
result := make(chan string)
kill := make(chan struct{})
+ defer close(result)
+ defer close(kill)
go startRoutines(result, kill, chunkedUrlsToCheck, workersCount, callback)
select {
case res := <-result:
kill <- struct{}{}
return res, nil
case _ = <-time.After(15 * time.Second):
kill <- struct{}{}
return "", &CallError{message: "Nothing found !!!"}
}
}
-func startRoutines(result chan string, ownDeath chan struct{}, chunkedUrlsToCheck <-chan []string, workersCount int, callback func(string) bool) {
+func startRoutines(result chan string, seppuku chan struct{}, chunkedUrlsToCheck <-chan []string, workersCount int, callback func(string) bool) {
+ defer close(seppuku)
poolOfWorkers := make(chan struct{}, workersCount)
for i := 0; i < workersCount; i++ {
poolOfWorkers <- struct{}{}
}
i := 0
killSliceRoutine := make(map[int]chan struct{})
for {
select {
- case _ = <-ownDeath:
+ case _ = <-seppuku:
for _, killer := range killSliceRoutine {
killer <- struct{}{}
+ close(killer)
}
return
case urlSlice := <-chunkedUrlsToCheck:
killSliceRoutine[i] = make(chan struct{})
go func(death chan struct{}, urlSlice []string) {
for _, url := range urlSlice {
select {
case _ = <-poolOfWorkers:
- stringMatch := make(chan string)
+ stringMatch := make(chan string, 2)
stringNotMatch := make(chan struct{})
go func(url string, stringMatch chan string, stringNotMatch chan struct{}) {
- if callback(url) {
- stringMatch <- url
- } else {
+ response, err := http.Get(url)
+ if err != nil {
stringNotMatch <- struct{}{}
+ } else {
+ defer response.Body.Close()
+ contents, err2 := ioutil.ReadAll(response.Body)
+ if err2 != nil {
+ stringNotMatch <- struct{}{}
+ return
+ }
+ stringMatch <- string(contents)
+ stringMatch <- url
}
}(url, stringMatch, stringNotMatch)
go func(stringMatch chan string, stringNotMatch chan struct{}) {
select {
case _ = <-time.After(3 * time.Second):
poolOfWorkers <- struct{}{}
- case match := <-stringMatch:
- result <- match
+ case httpContent := <-stringMatch:
+ if callback(httpContent) {
+ successURL := <-stringMatch
+ result <- successURL
+ } else {
+ poolOfWorkers <- struct{}{}
+ }
case _ = <-stringNotMatch:
poolOfWorkers <- struct{}{}
}
}(stringMatch, stringNotMatch)
case _ = <-death:
return
}
}
}(killSliceRoutine[i], urlSlice)
i++
}
}
-}
+}

Йосиф обнови решението на 11.12.2014 16:33 (преди над 3 години)

package main
import "time"
import "net/http"
import "io/ioutil"
type CallError struct {
message string
}
func (ce *CallError) Error() string {
return ce.message
}
func SeekAndDestroy(callback func(string) bool, chunkedUrlsToCheck <-chan []string, workersCount int) (str string, err error) {
if workersCount < 1 {
return "", &CallError{message: "Invalid value for \"workersCount\" !!!"}
} else if chunkedUrlsToCheck == nil {
return "", &CallError{message: "Uninitialised channel !!!"}
}
result := make(chan string)
- kill := make(chan struct{})
+ kill := make(chan struct{}, 1)
defer close(result)
defer close(kill)
go startRoutines(result, kill, chunkedUrlsToCheck, workersCount, callback)
select {
case res := <-result:
kill <- struct{}{}
+ if res == "nil" {
+ return "", &CallError{message: "Nothing found !!!"}
+ }
return res, nil
case _ = <-time.After(15 * time.Second):
kill <- struct{}{}
return "", &CallError{message: "Nothing found !!!"}
}
}
func startRoutines(result chan string, seppuku chan struct{}, chunkedUrlsToCheck <-chan []string, workersCount int, callback func(string) bool) {
- defer close(seppuku)
poolOfWorkers := make(chan struct{}, workersCount)
for i := 0; i < workersCount; i++ {
poolOfWorkers <- struct{}{}
}
i := 0
killSliceRoutine := make(map[int]chan struct{})
for {
select {
case _ = <-seppuku:
for _, killer := range killSliceRoutine {
killer <- struct{}{}
close(killer)
}
return
case urlSlice := <-chunkedUrlsToCheck:
+ if len(urlSlice) == 0 {
+ result <- "nil"
+ return
+ }
killSliceRoutine[i] = make(chan struct{})
go func(death chan struct{}, urlSlice []string) {
for _, url := range urlSlice {
select {
case _ = <-poolOfWorkers:
stringMatch := make(chan string, 2)
stringNotMatch := make(chan struct{})
go func(url string, stringMatch chan string, stringNotMatch chan struct{}) {
response, err := http.Get(url)
if err != nil {
stringNotMatch <- struct{}{}
} else {
defer response.Body.Close()
contents, err2 := ioutil.ReadAll(response.Body)
if err2 != nil {
stringNotMatch <- struct{}{}
return
}
stringMatch <- string(contents)
stringMatch <- url
}
}(url, stringMatch, stringNotMatch)
go func(stringMatch chan string, stringNotMatch chan struct{}) {
select {
case _ = <-time.After(3 * time.Second):
poolOfWorkers <- struct{}{}
case httpContent := <-stringMatch:
if callback(httpContent) {
successURL := <-stringMatch
result <- successURL
} else {
poolOfWorkers <- struct{}{}
}
case _ = <-stringNotMatch:
poolOfWorkers <- struct{}{}
}
}(stringMatch, stringNotMatch)
case _ = <-death:
return
}
}
}(killSliceRoutine[i], urlSlice)
i++
}
}
}