๐ธ๏ธ
ActoCrawler
ActoCrawler is a Swift Concurrency-powered crawler engine on top of Actomaton, with flexible customizability to create various HTML scrapers, image scrapers, etc.
Example
struct Output: Sendable
{
let nextLinksCount: Int
}
let htmlCrawler = await Crawler<Output, Void>.htmlScraper(
config: CrawlerConfig(
maxDepths: 10,
maxTotalRequests: 100,
timeoutPerRequest: 5,
userAgent: "ActoCrawler",
domainFilteringPolicy: .disallowedDomains([".*google.com*" /* ... */]),
domainQueueTable: [
".*example1.com*": .init(maxConcurrency: 1, delay: 0),
".*example2.com*": .init(maxConcurrency: 5, delay: 0.1 ... 0.5)
]
),
scrapeHTML: { response in
let html = response.data
let links = try html.select("a").map { try $0.attr("href") }
let nextRequests = links
.compactMap(URL.init(string:))
.map { UserRequest(url: $0) }
return (nextRequests, Output(nextLinksCount: nextRequests.count))
}
)
// Visit initial page.
htmlCrawler.visit(url: URL(string: "https://www.wikipedia.org")!)
// Observe crawl events.
for await event in htmlCrawler.events {
switch event {
case let .willCrawl(req):
print("Crawl : ๐ธ๏ธ [\(req.order)] [d=\(req.depth)] \(req.url)")
case let .didCrawl(req, .success(output)):
print("Output: โ
[\(req.order)] [d=\(req.depth)] \(req.url), nextLinksCount = \(output.nextLinksCount)")
case let .didCrawl(req, .failure(error)):
print("Output: โ [\(req.order)] [d=\(req.depth)] \(req.url), error = \(error)")
}
}
print("Output Done")
620 Oct 11, 2022
0 Mar 9, 2022
12 Dec 29, 2022
86 Jun 29, 2022
3 Sep 29, 2022
7 Nov 24, 2022
57 Dec 24, 2022
8 Aug 1, 2022
4 Aug 14, 2022
33 Oct 3, 2022
374 Jan 3, 2023
582 Dec 25, 2022
39 Dec 31, 2022
581 Dec 31, 2022
2.3k Jan 3, 2023
26 Dec 10, 2022
478 Dec 29, 2022
2k Dec 28, 2022