non-greedy повторители

Програмиране с Go

Курс във Факултета по Математика и Информатика към СУ

Решение на Concurrent Crawling от Илия Ячев

Обратно към всички решения

Към профила на Илия Ячев

Резултати

  • 8 точки от тестове
  • 0 бонус точки
  • 8 точки общо
  • 9 успешни тест(а)
  • 2 неуспешни тест(а)

Код

package main
import (
"errors"
"io"
"io/ioutil"
"net/http"
"time"
)
type result struct {
Req *http.Request
Found bool
}
func (r *result) Url() string {
return r.Req.URL.String()
}
func SeekAndDestroy(callback func(string) bool, chunkedUrlsToCheck <-chan []string, workersCount int) (string, error) {
if workersCount <= 0 {
return "", errors.New("workersCount is not positive")
}
queue := []*http.Request{}
timeout := time.After(15 * time.Second)
workers := 0
results := make(chan *result, workersCount)
activeRequests := make(map[*http.Request]bool, workersCount)
client := &http.Client{Timeout: 3 * time.Second, Transport: &http.Transport{}}
defer func() {
if c, ok := client.Transport.(*http.Transport); ok {
for req := range activeRequests {
c.CancelRequest(req)
}
}
}()
for {
select {
case urls, ok := <-chunkedUrlsToCheck:
if !ok {
return "", errors.New("urls chan was closed")
}
for _, rawurl := range urls {
if req, err := http.NewRequest("GET", rawurl, *new(io.Reader)); err == nil {
queue = append(queue, req)
}
}
for workers < workersCount && len(queue) != 0 {
workers++
activeRequests[queue[0]] = true
go testUrl(client, queue[0], callback, results)
queue = queue[1:] // memory leak - should I use containers/list instead?
}
case r := <-results:
delete(activeRequests, r.Req)
if r.Found {
return r.Url(), nil
}
if len(queue) != 0 {
activeRequests[queue[0]] = true
go testUrl(client, queue[0], callback, results)
queue = queue[1:]
} else {
workers--
}
case <-timeout:
return "", errors.New("timeout")
}
}
}
func testUrl(client *http.Client, req *http.Request, condition func(string) bool, results chan *result) {
body, err := get(client, req)
if err == nil {
results <- &result{Req: req, Found: condition(body)}
} else {
results <- &result{Req: req, Found: false}
}
}
func get(client *http.Client, req *http.Request) (result string, err error) {
res, err := client.Do(req)
if err != nil {
return
}
body, err := ioutil.ReadAll(res.Body)
res.Body.Close()
if err != nil {
return
}
result = string(body)
if res.StatusCode/100 != 2 {
return result, errors.New(res.Status)
}
return
}

Лог от изпълнението

[/tmp/go-build810962746/_/tmp/d20150111-16649-147k0kc/_test/d20150111-16649-147k0kc.test -test.run=TestWithNegativeWorkersCount -test.timeout=120s]
PASS
ok  	_/tmp/d20150111-16649-147k0kc	0.008s
[/tmp/go-build059095926/_/tmp/d20150111-16649-147k0kc/_test/d20150111-16649-147k0kc.test -test.run=TestWithZeroWorkersCount -test.timeout=120s]
PASS
ok  	_/tmp/d20150111-16649-147k0kc	0.008s
[/tmp/go-build291736398/_/tmp/d20150111-16649-147k0kc/_test/d20150111-16649-147k0kc.test -test.run=TestWithInvalidCallback -test.timeout=120s]
--- FAIL: TestWithInvalidCallback-2 (1.00 seconds)
	solution_test.go:43: Test exceeded allowed time of 1 seconds: parameter errors should be immediately returned (callback is nil)
FAIL
exit status 1
FAIL	_/tmp/d20150111-16649-147k0kc	1.006s
[/tmp/go-build842532904/_/tmp/d20150111-16649-147k0kc/_test/d20150111-16649-147k0kc.test -test.run=TestWithNilChannel -test.timeout=120s]
--- FAIL: TestWithNilChannel-2 (1.00 seconds)
	solution_test.go:43: Test exceeded allowed time of 1 seconds: parameter errors should be immediately returned (channel is uninitialized)
FAIL
exit status 1
FAIL	_/tmp/d20150111-16649-147k0kc	1.005s
[/tmp/go-build008400866/_/tmp/d20150111-16649-147k0kc/_test/d20150111-16649-147k0kc.test -test.run=TestWithClosedChannelWhenStarting -test.timeout=120s]
PASS
ok  	_/tmp/d20150111-16649-147k0kc	0.008s
[/tmp/go-build472705927/_/tmp/d20150111-16649-147k0kc/_test/d20150111-16649-147k0kc.test -test.run=TestWithClosedChannelMidway -test.timeout=120s]
PASS
ok  	_/tmp/d20150111-16649-147k0kc	5.006s
[/tmp/go-build739952414/_/tmp/d20150111-16649-147k0kc/_test/d20150111-16649-147k0kc.test -test.run=TestWhetherGlobalTimeoutIsHandled -test.timeout=120s]
PASS
ok  	_/tmp/d20150111-16649-147k0kc	15.006s
[/tmp/go-build633582201/_/tmp/d20150111-16649-147k0kc/_test/d20150111-16649-147k0kc.test -test.run=TestWithLoremIpsum -test.timeout=120s]
PASS
ok  	_/tmp/d20150111-16649-147k0kc	2.009s
[/tmp/go-build873072232/_/tmp/d20150111-16649-147k0kc/_test/d20150111-16649-147k0kc.test -test.run=TestIfTimeoutAndErrorCodesAreHonoured -test.timeout=120s]
PASS
ok  	_/tmp/d20150111-16649-147k0kc	8.009s
[/tmp/go-build352670434/_/tmp/d20150111-16649-147k0kc/_test/d20150111-16649-147k0kc.test -test.run=TestRaceCondition -test.timeout=120s]
PASS
ok  	_/tmp/d20150111-16649-147k0kc	1.007s
[/tmp/go-build175133481/_/tmp/d20150111-16649-147k0kc/_test/d20150111-16649-147k0kc.test -test.run=TestCloseChannelBeforeFinish -test.timeout=120s]
PASS
ok  	_/tmp/d20150111-16649-147k0kc	1.006s

История (9 версии и 5 коментара)

Илия обнови решението на 05.12.2014 06:59 (преди над 3 години)

+package main
+// Evans doesn't let me commit even though go fmt, run and test are fine

Илия обнови решението на 05.12.2014 07:09 (преди над 3 години)

package main
-// Evans doesn't let me commit even though go fmt, run and test are fine
+
+import (
+ // "errors"
+ // "io/ioutil"
+ // "net/http"
+ // "time"
+)
+
+// func SeekAndDestroy(callback func(string) bool, chunkedUrlsToCheck <-chan []string, workersCount int) (string, error) {
+// if workersCount <= 0 { // || chunkedUrlsToCheck != nil - empty channel is equal to nil
+// return "", errors.New("workersCount is not positive")
+// }
+// queue := []string{}
+// timeout := time.After(15 * time.Second)
+// workers := 0
+// result := make(chan string, workersCount)
+// // quit := make(chan struct{})
+// // defer close(quit)
+// for {
+// select {
+// case urls, ok := <-chunkedUrlsToCheck:
+// if !ok {
+// return "", errors.New("urls chan was closed")
+// }
+// queue := append(queue, urls...)
+// for workers < workersCount && len(queue) != 0 {
+// workers++
+// go testUrl(queue[0], callback, result)
+// queue = queue[1:] // memory leak - should I use containers/list instead?
+// }
+// case r := <-result:
+// if r != "" {
+// return r, nil
+// }
+// if len(queue) != 0 {
+// go testUrl(queue[0], callback, result)
+// queue = queue[1:]
+// } else {
+// workers--
+// }
+// case <-timeout:
+// return "", errors.New("timeout")
+// }
+// }
+// }
+// func testUrl(url string, condition func(string) bool, result chan string) {
+// // fmt.Println("Getting " + url)
+// body, err := get(url)
+// if err == nil && condition(body) {
+// result <- url
+// } else {
+// result <- ""
+// }
+// }
+// func get(url string) (result string, err error) {
+// res, err := (&http.Client{Timeout: 3 * time.Second}).Get(url)
+// // TODO: use Client.Transport.CancelRequest
+// if err != nil {
+// return
+// }
+// body, err := ioutil.ReadAll(res.Body)
+// res.Body.Close()
+// if err != nil {
+// return
+// }
+// result = string(body)
+// if res.StatusCode/100 != 2 {
+// return result, errors.New(res.Status)
+// }
+// return
+// }

Илия обнови решението на 05.12.2014 07:10 (преди над 3 години)

package main
import (
// "errors"
// "io/ioutil"
// "net/http"
// "time"
)
// func SeekAndDestroy(callback func(string) bool, chunkedUrlsToCheck <-chan []string, workersCount int) (string, error) {
// if workersCount <= 0 { // || chunkedUrlsToCheck != nil - empty channel is equal to nil
// return "", errors.New("workersCount is not positive")
// }
// queue := []string{}
// timeout := time.After(15 * time.Second)
// workers := 0
// result := make(chan string, workersCount)
// // quit := make(chan struct{})
// // defer close(quit)
// for {
// select {
// case urls, ok := <-chunkedUrlsToCheck:
// if !ok {
// return "", errors.New("urls chan was closed")
// }
// queue := append(queue, urls...)
// for workers < workersCount && len(queue) != 0 {
// workers++
// go testUrl(queue[0], callback, result)
// queue = queue[1:] // memory leak - should I use containers/list instead?
// }
// case r := <-result:
// if r != "" {
// return r, nil
// }
// if len(queue) != 0 {
// go testUrl(queue[0], callback, result)
// queue = queue[1:]
// } else {
// workers--
// }
// case <-timeout:
// return "", errors.New("timeout")
// }
// }
// }
-// func testUrl(url string, condition func(string) bool, result chan string) {
-// // fmt.Println("Getting " + url)
+func testUrl(url string, condition func(string) bool, result chan string) {
-// body, err := get(url)
+ // fmt.Println("Getting " + url)
-// if err == nil && condition(body) {
+ body, err := get(url)
-// result <- url
+ if err == nil && condition(body) {
-// } else {
+ result <- url
-// result <- ""
+ } else {
-// }
+ result <- ""
-// }
+ }
-// func get(url string) (result string, err error) {
+}
-// res, err := (&http.Client{Timeout: 3 * time.Second}).Get(url)
+func get(url string) (result string, err error) {
-// // TODO: use Client.Transport.CancelRequest
+ // res, err := (&http.Client{Timeout: 3 * time.Second}).Get(url)
-// if err != nil {
+ // // TODO: use Client.Transport.CancelRequest
-// return
+ // if err != nil {
-// }
+ // return
-// body, err := ioutil.ReadAll(res.Body)
+ // }
-// res.Body.Close()
+ // body, err := ioutil.ReadAll(res.Body)
-// if err != nil {
+ // res.Body.Close()
-// return
+ // if err != nil {
-// }
+ // return
-// result = string(body)
+ // }
-// if res.StatusCode/100 != 2 {
+ // result = string(body)
-// return result, errors.New(res.Status)
+ // if res.StatusCode/100 != 2 {
-// }
+ // return result, errors.New(res.Status)
-// return
+ // }
-// }
+ return
+}

Илия обнови решението на 05.12.2014 07:16 (преди над 3 години)

package main
import (
- // "errors"
+ "errors"
// "io/ioutil"
// "net/http"
- // "time"
+ "time"
)
-// func SeekAndDestroy(callback func(string) bool, chunkedUrlsToCheck <-chan []string, workersCount int) (string, error) {
-// if workersCount <= 0 { // || chunkedUrlsToCheck != nil - empty channel is equal to nil
-// return "", errors.New("workersCount is not positive")
-// }
-// queue := []string{}
-// timeout := time.After(15 * time.Second)
-// workers := 0
-// result := make(chan string, workersCount)
-// // quit := make(chan struct{})
-// // defer close(quit)
-// for {
-// select {
-// case urls, ok := <-chunkedUrlsToCheck:
-// if !ok {
-// return "", errors.New("urls chan was closed")
-// }
-// queue := append(queue, urls...)
-// for workers < workersCount && len(queue) != 0 {
-// workers++
-// go testUrl(queue[0], callback, result)
-// queue = queue[1:] // memory leak - should I use containers/list instead?
-// }
-// case r := <-result:
-// if r != "" {
-// return r, nil
-// }
-// if len(queue) != 0 {
-// go testUrl(queue[0], callback, result)
-// queue = queue[1:]
-// } else {
-// workers--
-// }
-// case <-timeout:
-// return "", errors.New("timeout")
-// }
-// }
-// }
+func SeekAndDestroy(callback func(string) bool, chunkedUrlsToCheck <-chan []string, workersCount int) (string, error) {
+ if workersCount <= 0 { // || chunkedUrlsToCheck != nil - empty channel is equal to nil
+ return "", errors.New("workersCount is not positive")
+ }
+ queue := []string{}
+ timeout := time.After(15 * time.Second)
+ workers := 0
+ result := make(chan string, workersCount)
+ // quit := make(chan struct{})
+ // defer close(quit)
+ for {
+ select {
+ case urls, ok := <-chunkedUrlsToCheck:
+ if !ok {
+ return "", errors.New("urls chan was closed")
+ }
+ queue := append(queue, urls...)
+ for workers < workersCount && len(queue) != 0 {
+ workers++
+ go testUrl(queue[0], callback, result)
+ queue = queue[1:] // memory leak - should I use containers/list instead?
+ }
+ case r := <-result:
+ if r != "" {
+ return r, nil
+ }
+ if len(queue) != 0 {
+ go testUrl(queue[0], callback, result)
+ queue = queue[1:]
+ } else {
+ workers--
+ }
+ case <-timeout:
+ return "", errors.New("timeout")
+ }
+ }
+}
func testUrl(url string, condition func(string) bool, result chan string) {
// fmt.Println("Getting " + url)
body, err := get(url)
if err == nil && condition(body) {
result <- url
} else {
result <- ""
}
}
func get(url string) (result string, err error) {
// res, err := (&http.Client{Timeout: 3 * time.Second}).Get(url)
- // // TODO: use Client.Transport.CancelRequest
+ // TODO: use Client.Transport.CancelRequest
// if err != nil {
// return
// }
// body, err := ioutil.ReadAll(res.Body)
// res.Body.Close()
// if err != nil {
// return
// }
// result = string(body)
// if res.StatusCode/100 != 2 {
// return result, errors.New(res.Status)
// }
return
}

Илия обнови решението на 05.12.2014 07:21 (преди над 3 години)

package main
import (
"errors"
// "io/ioutil"
- // "net/http"
+ "net/http"
"time"
)
func SeekAndDestroy(callback func(string) bool, chunkedUrlsToCheck <-chan []string, workersCount int) (string, error) {
if workersCount <= 0 { // || chunkedUrlsToCheck != nil - empty channel is equal to nil
return "", errors.New("workersCount is not positive")
}
queue := []string{}
timeout := time.After(15 * time.Second)
workers := 0
result := make(chan string, workersCount)
// quit := make(chan struct{})
// defer close(quit)
for {
select {
case urls, ok := <-chunkedUrlsToCheck:
if !ok {
return "", errors.New("urls chan was closed")
}
queue := append(queue, urls...)
for workers < workersCount && len(queue) != 0 {
workers++
go testUrl(queue[0], callback, result)
queue = queue[1:] // memory leak - should I use containers/list instead?
}
case r := <-result:
if r != "" {
return r, nil
}
if len(queue) != 0 {
go testUrl(queue[0], callback, result)
queue = queue[1:]
} else {
workers--
}
case <-timeout:
return "", errors.New("timeout")
}
}
}
func testUrl(url string, condition func(string) bool, result chan string) {
// fmt.Println("Getting " + url)
body, err := get(url)
if err == nil && condition(body) {
result <- url
} else {
result <- ""
}
}
func get(url string) (result string, err error) {
- // res, err := (&http.Client{Timeout: 3 * time.Second}).Get(url)
+ client := &http.Client{}//Timeout: 3 * time.Second}
+ _, err = client.Get(url)
// TODO: use Client.Transport.CancelRequest
// if err != nil {
// return
// }
// body, err := ioutil.ReadAll(res.Body)
// res.Body.Close()
// if err != nil {
// return
// }
// result = string(body)
// if res.StatusCode/100 != 2 {
// return result, errors.New(res.Status)
// }
return
}

Илия обнови решението на 05.12.2014 07:22 (преди над 3 години)

package main
import (
"errors"
- // "io/ioutil"
+ "io/ioutil"
"net/http"
"time"
)
func SeekAndDestroy(callback func(string) bool, chunkedUrlsToCheck <-chan []string, workersCount int) (string, error) {
if workersCount <= 0 { // || chunkedUrlsToCheck != nil - empty channel is equal to nil
return "", errors.New("workersCount is not positive")
}
queue := []string{}
timeout := time.After(15 * time.Second)
workers := 0
result := make(chan string, workersCount)
// quit := make(chan struct{})
// defer close(quit)
for {
select {
case urls, ok := <-chunkedUrlsToCheck:
if !ok {
return "", errors.New("urls chan was closed")
}
queue := append(queue, urls...)
for workers < workersCount && len(queue) != 0 {
workers++
go testUrl(queue[0], callback, result)
queue = queue[1:] // memory leak - should I use containers/list instead?
}
case r := <-result:
if r != "" {
return r, nil
}
if len(queue) != 0 {
go testUrl(queue[0], callback, result)
queue = queue[1:]
} else {
workers--
}
case <-timeout:
return "", errors.New("timeout")
}
}
}
func testUrl(url string, condition func(string) bool, result chan string) {
// fmt.Println("Getting " + url)
body, err := get(url)
if err == nil && condition(body) {
result <- url
} else {
result <- ""
}
}
func get(url string) (result string, err error) {
- client := &http.Client{}//Timeout: 3 * time.Second}
- _, err = client.Get(url)
+ client := &http.Client{} //Timeout: 3 * time.Second}
+ res, err := client.Get(url)
// TODO: use Client.Transport.CancelRequest
- // if err != nil {
- // return
- // }
- // body, err := ioutil.ReadAll(res.Body)
- // res.Body.Close()
- // if err != nil {
- // return
- // }
- // result = string(body)
- // if res.StatusCode/100 != 2 {
- // return result, errors.New(res.Status)
- // }
+ if err != nil {
+ return
+ }
+ body, err := ioutil.ReadAll(res.Body)
+ res.Body.Close()
+ if err != nil {
+ return
+ }
+ result = string(body)
+ if res.StatusCode/100 != 2 {
+ return result, errors.New(res.Status)
+ }
return
}

Бях го сложил в коментар, понеже когато го commit-вах Кирил още не беше ъпдейтнал go до 1.3 и evans ми казваше, че имам синтактична грешка. Понеже сегашната ми имплементация не убива горутините когато намери резултат планирах да си пренапиша задачата и затова не съм направил commit само с откоментирането на Timeout-a.

Илия обнови решението на 11.12.2014 15:22 (преди над 3 години)

package main
import (
"errors"
"io/ioutil"
"net/http"
"time"
)
func SeekAndDestroy(callback func(string) bool, chunkedUrlsToCheck <-chan []string, workersCount int) (string, error) {
if workersCount <= 0 { // || chunkedUrlsToCheck != nil - empty channel is equal to nil
return "", errors.New("workersCount is not positive")
}
queue := []string{}
timeout := time.After(15 * time.Second)
workers := 0
result := make(chan string, workersCount)
// quit := make(chan struct{})
// defer close(quit)
for {
select {
case urls, ok := <-chunkedUrlsToCheck:
if !ok {
return "", errors.New("urls chan was closed")
}
queue := append(queue, urls...)
for workers < workersCount && len(queue) != 0 {
workers++
go testUrl(queue[0], callback, result)
queue = queue[1:] // memory leak - should I use containers/list instead?
}
case r := <-result:
if r != "" {
return r, nil
}
if len(queue) != 0 {
go testUrl(queue[0], callback, result)
queue = queue[1:]
} else {
workers--
}
case <-timeout:
return "", errors.New("timeout")
}
}
}
func testUrl(url string, condition func(string) bool, result chan string) {
// fmt.Println("Getting " + url)
body, err := get(url)
if err == nil && condition(body) {
result <- url
} else {
result <- ""
}
}
func get(url string) (result string, err error) {
- client := &http.Client{} //Timeout: 3 * time.Second}
+ client := &http.Client{Timeout: 3 * time.Second}
res, err := client.Get(url)
// TODO: use Client.Transport.CancelRequest
if err != nil {
return
}
body, err := ioutil.ReadAll(res.Body)
res.Body.Close()
if err != nil {
return
}
result = string(body)
if res.StatusCode/100 != 2 {
return result, errors.New(res.Status)
}
return
}

Илия обнови решението на 11.12.2014 16:03 (преди над 3 години)

package main
import (
"errors"
+ "fmt"
+ "io"
"io/ioutil"
"net/http"
"time"
)
+type result struct {
+ Req *http.Request
+ Found bool
+}
+
+func (r *result) Url() string {
+ return r.Req.URL.String()
+}
+
func SeekAndDestroy(callback func(string) bool, chunkedUrlsToCheck <-chan []string, workersCount int) (string, error) {
- if workersCount <= 0 { // || chunkedUrlsToCheck != nil - empty channel is equal to nil
+ if workersCount <= 0 {
return "", errors.New("workersCount is not positive")
}
- queue := []string{}
+ queue := []*http.Request{}
timeout := time.After(15 * time.Second)
workers := 0
- result := make(chan string, workersCount)
- // quit := make(chan struct{})
- // defer close(quit)
+ results := make(chan *result, workersCount)
+ activeRequests := make(map[*http.Request]bool, workersCount)
+ client := &http.Client{Timeout: 3 * time.Second, Transport: &http.Transport{}}
+
+ defer func() {
+ if c, ok := client.Transport.(*http.Transport); ok {
+ for req := range activeRequests {
+ c.CancelRequest(req)
+ }
+ }
+ }()
+
for {
select {
case urls, ok := <-chunkedUrlsToCheck:
if !ok {
return "", errors.New("urls chan was closed")
}
- queue := append(queue, urls...)
+ for _, rawurl := range urls {
+ if req, err := http.NewRequest("GET", rawurl, *new(io.Reader)); err == nil {
+ queue = append(queue, req)
+ }
+ }
for workers < workersCount && len(queue) != 0 {
workers++
- go testUrl(queue[0], callback, result)
+ activeRequests[queue[0]] = true
+ go testUrl(client, queue[0], callback, results)
queue = queue[1:] // memory leak - should I use containers/list instead?
}
- case r := <-result:
- if r != "" {
- return r, nil
+ case r := <-results:
+ delete(activeRequests, r.Req)
+ if r.Found {
+ return r.Url(), nil
}
if len(queue) != 0 {
- go testUrl(queue[0], callback, result)
+ activeRequests[queue[0]] = true
+ go testUrl(client, queue[0], callback, results)
queue = queue[1:]
} else {
workers--
}
case <-timeout:
return "", errors.New("timeout")
}
}
}
-func testUrl(url string, condition func(string) bool, result chan string) {
- // fmt.Println("Getting " + url)
- body, err := get(url)
- if err == nil && condition(body) {
- result <- url
+func testUrl(client *http.Client, req *http.Request, condition func(string) bool, results chan *result) {
+ fmt.Println("Getting " + req.URL.String())
+ body, err := get(client, req)
+ if err == nil {
+ results <- &result{Req: req, Found: condition(body)}
} else {
- result <- ""
+ results <- &result{Req: req, Found: false}
+ fmt.Println("Error getting " + req.URL.String())
}
}
-func get(url string) (result string, err error) {
- client := &http.Client{Timeout: 3 * time.Second}
- res, err := client.Get(url)
- // TODO: use Client.Transport.CancelRequest
+func get(client *http.Client, req *http.Request) (result string, err error) {
+ res, err := client.Do(req)
if err != nil {
return
}
body, err := ioutil.ReadAll(res.Body)
res.Body.Close()
if err != nil {
return
}
result = string(body)
if res.StatusCode/100 != 2 {
return result, errors.New(res.Status)
}
return
}

Илия обнови решението на 11.12.2014 16:07 (преди над 3 години)

package main
import (
"errors"
- "fmt"
"io"
"io/ioutil"
"net/http"
"time"
)
type result struct {
Req *http.Request
Found bool
}
func (r *result) Url() string {
return r.Req.URL.String()
}
func SeekAndDestroy(callback func(string) bool, chunkedUrlsToCheck <-chan []string, workersCount int) (string, error) {
if workersCount <= 0 {
return "", errors.New("workersCount is not positive")
}
queue := []*http.Request{}
timeout := time.After(15 * time.Second)
workers := 0
results := make(chan *result, workersCount)
activeRequests := make(map[*http.Request]bool, workersCount)
client := &http.Client{Timeout: 3 * time.Second, Transport: &http.Transport{}}
defer func() {
if c, ok := client.Transport.(*http.Transport); ok {
for req := range activeRequests {
c.CancelRequest(req)
}
}
}()
for {
select {
case urls, ok := <-chunkedUrlsToCheck:
if !ok {
return "", errors.New("urls chan was closed")
}
for _, rawurl := range urls {
if req, err := http.NewRequest("GET", rawurl, *new(io.Reader)); err == nil {
queue = append(queue, req)
}
}
for workers < workersCount && len(queue) != 0 {
workers++
activeRequests[queue[0]] = true
go testUrl(client, queue[0], callback, results)
queue = queue[1:] // memory leak - should I use containers/list instead?
}
case r := <-results:
delete(activeRequests, r.Req)
if r.Found {
return r.Url(), nil
}
if len(queue) != 0 {
activeRequests[queue[0]] = true
go testUrl(client, queue[0], callback, results)
queue = queue[1:]
} else {
workers--
}
case <-timeout:
return "", errors.New("timeout")
}
}
}
func testUrl(client *http.Client, req *http.Request, condition func(string) bool, results chan *result) {
- fmt.Println("Getting " + req.URL.String())
body, err := get(client, req)
if err == nil {
results <- &result{Req: req, Found: condition(body)}
} else {
results <- &result{Req: req, Found: false}
- fmt.Println("Error getting " + req.URL.String())
}
}
func get(client *http.Client, req *http.Request) (result string, err error) {
res, err := client.Do(req)
if err != nil {
return
}
body, err := ioutil.ReadAll(res.Body)
res.Body.Close()
if err != nil {
return
}
result = string(body)
if res.StatusCode/100 != 2 {
return result, errors.New(res.Status)
}
return
}

Сега само се надявам новата имплементация да не е много бъгава. :laughing:

Очевидно щеше да е по-добре да си направя wrapper на клиента, който се занимава с повечето работа, но пренаписването отново непосредствено преди крайния срок е опасно.

А 8ми декември не помогна да си пишем домашните навреме.