@@ -50,15 +50,15 @@ and easy to use.
5050Start spidering from a URL:
5151
5252``` ruby
53- Spidr .start_at(' http ://tenderlovemaking.com /' ) do |agent |
53+ Spidr .start_at(' https ://www.ruby-lang.org/en /' ) do |agent |
5454 # ...
5555end
5656```
5757
5858Spider a host:
5959
6060``` ruby
61- Spidr .host(' solnic.dev ' ) do |agent |
61+ Spidr .host(' www.ruby-lang.org ' ) do |agent |
6262 # ...
6363end
6464```
7474Spider a site:
7575
7676``` ruby
77- Spidr .site(' http ://www.rubyflow.com /' ) do |agent |
77+ Spidr .site(' https ://www.ruby-lang.org /' ) do |agent |
7878 # ...
7979end
8080```
8181
8282Spider multiple hosts:
8383
8484``` ruby
85- Spidr .start_at(' http ://company.com/ ' , hosts: [' company.com ' , /.*\. company \. com \z / ]) do |agent |
85+ Spidr .start_at(' https ://www.ruby-lang.org/en/ ' , hosts: [' ruby-lang.org ' , /.*\. ruby-lang \. org \z / ]) do |agent |
8686 # ...
8787end
8888```
8989
9090Do not spider certain links:
9191
9292``` ruby
93- Spidr .site(' http ://company.com /' , ignore_links: [%r{\A /blog/} ]) do |agent |
93+ Spidr .site(' https ://www.ruby-lang.org /' , ignore_links: [%r{\A /blog/} ]) do |agent |
9494 # ...
9595end
9696```
9797
9898Do not spider links on certain ports:
9999
100100``` ruby
101- Spidr .site(' http ://company.com /' , ignore_ports: [8000 , 8010 , 8080 ]) do |agent |
101+ Spidr .site(' https ://www.ruby-lang.org /' , ignore_ports: [8000 , 8010 , 8080 ]) do |agent |
102102 # ...
103103end
104104```
105105
106106Do not spider links blacklisted in robots.txt:
107107
108108``` ruby
109- Spidr .site(' http ://company.com /' , robots: true ) do |agent |
109+ Spidr .site(' https ://www.ruby-lang.org /' , robots: true ) do |agent |
110110 # ...
111111end
112112```
113113
114114Print out visited URLs:
115115
116116``` ruby
117- Spidr .site(' http ://www.rubyinside.com /' ) do |spider |
117+ Spidr .site(' https ://www.ruby-lang.org /' ) do |spider |
118118 spider.every_url { |url | puts url }
119119end
120120```
@@ -124,7 +124,7 @@ Build a URL map of a site:
124124``` ruby
125125url_map = Hash .new { |hash ,key | hash[key] = [] }
126126
127- Spidr .site(' http ://intranet.com /' ) do |spider |
127+ Spidr .site(' https ://www.ruby-lang.org /' ) do |spider |
128128 spider.every_link do |origin ,dest |
129129 url_map[dest] << origin
130130 end
134134Print out the URLs that could not be requested:
135135
136136``` ruby
137- Spidr .site(' http ://company.com /' ) do |spider |
137+ Spidr .site(' https ://www.ruby-lang.org /' ) do |spider |
138138 spider.every_failed_url { |url | puts url }
139139end
140140```
@@ -144,7 +144,7 @@ Finds all pages which have broken links:
144144``` ruby
145145url_map = Hash .new { |hash ,key | hash[key] = [] }
146146
147- spider = Spidr .site(' http ://intranet.com /' ) do |spider |
147+ spider = Spidr .site(' https ://www.ruby-lang.org /' ) do |spider |
148148 spider.every_link do |origin ,dest |
149149 url_map[dest] << origin
150150 end
160160Search HTML and XML pages:
161161
162162``` ruby
163- Spidr .site(' http ://company.com /' ) do |spider |
163+ Spidr .site(' https ://www.ruby-lang.org /' ) do |spider |
164164 spider.every_page do |page |
165165 puts " >>> #{ page.url } "
166166
187187Print out every HTTP redirect:
188188
189189``` ruby
190- Spidr .host(' company.com ' ) do |spider |
190+ Spidr .host(' www.ruby-lang.org ' ) do |spider |
191191 spider.every_redirect_page do |page |
192192 puts " #{ page.url } -> #{ page.headers[' Location' ] } "
193193 end
@@ -199,7 +199,7 @@ Find what kinds of web servers a host is using, by accessing the headers:
199199``` ruby
200200servers = Set []
201201
202- Spidr .host(' company.com ' ) do |spider |
202+ Spidr .host(' www.ruby-lang.org ' ) do |spider |
203203 spider.all_headers do |headers |
204204 servers << headers[' server' ]
205205 end
209209Pause the spider on a forbidden page:
210210
211211``` ruby
212- Spidr .host(' company.com ' ) do |spider |
212+ Spidr .host(' www.ruby-lang.org ' ) do |spider |
213213 spider.every_forbidden_page do |page |
214214 spider.pause!
215215 end
219219Skip the processing of a page:
220220
221221``` ruby
222- Spidr .host(' company.com ' ) do |spider |
222+ Spidr .host(' www.ruby-lang.org ' ) do |spider |
223223 spider.every_missing_page do |page |
224224 spider.skip_page!
225225 end
229229Skip the processing of links:
230230
231231``` ruby
232- Spidr .host(' company.com ' ) do |spider |
232+ Spidr .host(' www.ruby-lang.org ' ) do |spider |
233233 spider.every_url do |url |
234234 if url.path.split(' /' ).find { |dir | dir.to_i > 1000 }
235235 spider.skip_link!
0 commit comments