Web Scraping Fridays - Revision history

Mike Chelen: /* Schedule */ .

2015-08-11T23:50:42Z

Schedule: .

← Older revision		Revision as of 23:50, 11 August 2015
Line 1:		Line 1:
	= Schedule =		= Schedule =
	- Aug 14
	- Aug 28		* Aug 14
	- Sep 11		* Aug 28
	- Sep 25		* Sep 11
			* Sep 25

	= CFAA =		= CFAA =

Mike Chelen: .

2015-08-11T23:48:24Z

← Older revision		Revision as of 23:48, 11 August 2015
Line 1:		Line 1:
			= Schedule =
			- Aug 14
			- Aug 28
			- Sep 11
			- Sep 25

	= CFAA =		= CFAA =

Mike Chelen: /* Examples */ .

2015-07-18T02:19:18Z

Examples: .

← Older revision		Revision as of 02:19, 18 July 2015
Line 9:		Line 9:
	wget https://www.congress.gov/search?q={%22source%22%3A%22legislation%22%2C%22congress%22%3A%22114%22}&pageSize=250		wget https://www.congress.gov/search?q={%22source%22%3A%22legislation%22%2C%22congress%22%3A%22114%22}&pageSize=250

	for each in `cat ../search\?q\=%7B%22source%22%3A%22legislation%22%2C%22congress%22%3A%22114%22%7D\&pageSize\=250 \| sed -n "s/.H\.R\.$[0-9]\{1,\}$./\1/p"`; do { wget https://www.congress.gov/bill/114th-congress/house-bill/$each/text?format=txt; } done;		for each in `cat ./search\?q\=%7B%22source%22%3A%22legislation%22%2C%22congress%22%3A%22114%22%7D\&pageSize\=250 \| sed -n "s/.H\.R\.$[0-9]\{1,\}$./\1/p"`; do { wget https://www.congress.gov/bill/114th-congress/house-bill/$each/text?format=txt; } done;

	cat rh\?format\=txt \| sed "/.pre>/{s/.$pre>$/FOUNDME:\1/;:a;N;s/^$FOUNDME:pre>\n\(\\|FOUNDME:.\n$\{0,\}\)$.$/\1FOUNDMORE:\3/;;s/FOUNDMORE/FOUNDME/;ta}"\| grep FOUNDME \| sed "/.div>/{s/.$div>$//;:a;N;s/.*//;;s/FOUNDME//;ta;}" \| less		cat rh\?format\=txt \| sed "/.pre>/{s/.$pre>$/FOUNDME:\1/;:a;N;s/^$FOUNDME:pre>\n\(\\|FOUNDME:.\n$\{0,\}\)$.$/\1FOUNDMORE:\3/;;s/FOUNDMORE/FOUNDME/;ta}"\| grep FOUNDME \| sed "/.div>/{s/.$div>$//;:a;N;s/.*//;;s/FOUNDME//;ta;}" \| less

Mike Chelen: /* Examples */ .

2015-07-18T02:18:13Z

Examples: .

← Older revision		Revision as of 02:18, 18 July 2015
Line 9:		Line 9:
	wget https://www.congress.gov/search?q={%22source%22%3A%22legislation%22%2C%22congress%22%3A%22114%22}&pageSize=250		wget https://www.congress.gov/search?q={%22source%22%3A%22legislation%22%2C%22congress%22%3A%22114%22}&pageSize=250

	for each in `cat ../search\?q\=%7B%22source%22%3A%22legislation%22%2C%22congress%22%3A%22114%22%7D\&pageSize\=250 \| sed -n "s/.H\.R\.$[0-9]\{1,\}$./\1/p"`; do { wget https://www.congress.gov/bill/114th-congress/house-bill/$each; } done;		for each in `cat ../search\?q\=%7B%22source%22%3A%22legislation%22%2C%22congress%22%3A%22114%22%7D\&pageSize\=250 \| sed -n "s/.H\.R\.$[0-9]\{1,\}$./\1/p"`; do { wget https://www.congress.gov/bill/114th-congress/house-bill/$each/text?format=txt; } done;

	cat rh\?format\=txt \| sed "/.pre>/{s/.$pre>$/FOUNDME:\1/;:a;N;s/^$FOUNDME:pre>\n\(\\|FOUNDME:.\n$\{0,\}\)$.$/\1FOUNDMORE:\3/;;s/FOUNDMORE/FOUNDME/;ta}"\| grep FOUNDME \| sed "/.div>/{s/.$div>$//;:a;N;s/.*//;;s/FOUNDME//;ta;}" \| less		cat rh\?format\=txt \| sed "/.pre>/{s/.$pre>$/FOUNDME:\1/;:a;N;s/^$FOUNDME:pre>\n\(\\|FOUNDME:.\n$\{0,\}\)$.$/\1FOUNDMORE:\3/;;s/FOUNDMORE/FOUNDME/;ta}"\| grep FOUNDME \| sed "/.div>/{s/.$div>$//;:a;N;s/.*//;;s/FOUNDME//;ta;}" \| less

Julialongtin: another example

2015-07-18T01:46:13Z

another example

← Older revision		Revision as of 01:46, 18 July 2015
Line 12:		Line 12:

	cat rh\?format\=txt \| sed "/.pre>/{s/.$pre>$/FOUNDME:\1/;:a;N;s/^$FOUNDME:pre>\n\(\\|FOUNDME:.\n$\{0,\}\)$.$/\1FOUNDMORE:\3/;;s/FOUNDMORE/FOUNDME/;ta}"\| grep FOUNDME \| sed "/.div>/{s/.$div>$//;:a;N;s/.*//;;s/FOUNDME//;ta;}" \| less		cat rh\?format\=txt \| sed "/.pre>/{s/.$pre>$/FOUNDME:\1/;:a;N;s/^$FOUNDME:pre>\n\(\\|FOUNDME:.\n$\{0,\}\)$.$/\1FOUNDMORE:\3/;;s/FOUNDMORE/FOUNDME/;ta}"\| grep FOUNDME \| sed "/.div>/{s/.$div>$//;:a;N;s/.*//;;s/FOUNDME//;ta;}" \| less

			cat rh\?format\=txt \| sed "/.pre>/{s/.$pre>$/FOUNDME:\1/;:a;N;s/^$FOUNDME:pre>\n\(\\|FOUNDME:.\n$\{0,\}\)$.$/\1FOUNDME:\3/;ba}"\| sed -n "s/FOUNDME://p" \| sed "/.div>/{s/.$div>$//;:a;N;s/.*//;ba;}" \| less
	</code>		</code>

	= Resources =		= Resources =

Mike Chelen: /* Examples */ .

2015-07-18T01:36:46Z

Examples: .

← Older revision		Revision as of 01:36, 18 July 2015
Line 7:		Line 7:
	= Examples =		= Examples =
	<code>		<code>
			wget https://www.congress.gov/search?q={%22source%22%3A%22legislation%22%2C%22congress%22%3A%22114%22}&pageSize=250

	for each in `cat ../search\?q\=%7B%22source%22%3A%22legislation%22%2C%22congress%22%3A%22114%22%7D\&pageSize\=250 \| sed -n "s/.H\.R\.$[0-9]\{1,\}$./\1/p"`; do { wget https://www.congress.gov/bill/114th-congress/house-bill/$each; } done;		for each in `cat ../search\?q\=%7B%22source%22%3A%22legislation%22%2C%22congress%22%3A%22114%22%7D\&pageSize\=250 \| sed -n "s/.H\.R\.$[0-9]\{1,\}$./\1/p"`; do { wget https://www.congress.gov/bill/114th-congress/house-bill/$each; } done;

Mike Chelen: /* Examples */ .

2015-07-18T01:36:18Z

Examples: .

← Older revision		Revision as of 01:36, 18 July 2015
Line 9:		Line 9:
	for each in `cat ../search\?q\=%7B%22source%22%3A%22legislation%22%2C%22congress%22%3A%22114%22%7D\&pageSize\=250 \| sed -n "s/.H\.R\.$[0-9]\{1,\}$./\1/p"`; do { wget https://www.congress.gov/bill/114th-congress/house-bill/$each; } done;		for each in `cat ../search\?q\=%7B%22source%22%3A%22legislation%22%2C%22congress%22%3A%22114%22%7D\&pageSize\=250 \| sed -n "s/.H\.R\.$[0-9]\{1,\}$./\1/p"`; do { wget https://www.congress.gov/bill/114th-congress/house-bill/$each; } done;

	cat rh\?format\=txt \| sed "/.pre>/{s/.$pre>$/FOUNDME:\1/;:a;N;s/^$FOUNDME:pre>\n\(\\|FOUNDME:.\n$\{0,\}\)$.$/\1FOUNDMORE:\3/;;s/FOUNDMORE/FOUNDME/;ta~~;:b~~}"\| grep FOUNDME \| sed "/.div>/{s/.$div>$//;:a;N;s/.*//;;s/FOUNDME//;ta;}" \| less		cat rh\?format\=txt \| sed "/.pre>/{s/.$pre>$/FOUNDME:\1/;:a;N;s/^$FOUNDME:pre>\n\(\\|FOUNDME:.\n$\{0,\}\)$.$/\1FOUNDMORE:\3/;;s/FOUNDMORE/FOUNDME/;ta}"\| grep FOUNDME \| sed "/.div>/{s/.$div>$//;:a;N;s/.*//;;s/FOUNDME//;ta;}" \| less
	</code>		</code>

	= Resources =		= Resources =

Julialongtin: another example

2015-07-18T01:28:24Z

another example

← Older revision		Revision as of 01:28, 18 July 2015
Line 8:		Line 8:
	<code>		<code>
	for each in `cat ../search\?q\=%7B%22source%22%3A%22legislation%22%2C%22congress%22%3A%22114%22%7D\&pageSize\=250 \| sed -n "s/.H\.R\.$[0-9]\{1,\}$./\1/p"`; do { wget https://www.congress.gov/bill/114th-congress/house-bill/$each; } done;		for each in `cat ../search\?q\=%7B%22source%22%3A%22legislation%22%2C%22congress%22%3A%22114%22%7D\&pageSize\=250 \| sed -n "s/.H\.R\.$[0-9]\{1,\}$./\1/p"`; do { wget https://www.congress.gov/bill/114th-congress/house-bill/$each; } done;

			cat rh\?format\=txt \| sed "/.pre>/{s/.$pre>$/FOUNDME:\1/;:a;N;s/^$FOUNDME:pre>\n\(\\|FOUNDME:.\n$\{0,\}\)$.$/\1FOUNDMORE:\3/;;s/FOUNDMORE/FOUNDME/;ta;:b}"\| grep FOUNDME \| sed "/.div>/{s/.$div>$//;:a;N;s/.*//;;s/FOUNDME//;ta;}" \| less
	</code>		</code>

	= Resources =		= Resources =

Mike Chelen: /* Examples */ foo

2015-07-18T01:24:43Z

Examples: foo

← Older revision		Revision as of 01:24, 18 July 2015
Line 6:		Line 6:

	= Examples =		= Examples =
	<~~syntaxhighlight lang="bash"~~>		<code>
	for each in `cat ../search\?q\=%7B%22source%22%3A%22legislation%22%2C%22congress%22%3A%22114%22%7D\&pageSize\=250 \| sed -n "s/.H\.R\.$[0-9]\{1,\}$./\1/p"`; do { wget https://www.congress.gov/bill/114th-congress/house-bill/$each; } done;		for each in `cat ../search\?q\=%7B%22source%22%3A%22legislation%22%2C%22congress%22%3A%22114%22%7D\&pageSize\=250 \| sed -n "s/.H\.R\.$[0-9]\{1,\}$./\1/p"`; do { wget https://www.congress.gov/bill/114th-congress/house-bill/$each; } done;
	</~~syntaxhighlight~~>		</code>

	= Resources =		= Resources =

Mike Chelen: /* Examples */ foo

2015-07-18T01:17:32Z

Examples: foo

← Older revision		Revision as of 01:17, 18 July 2015
Line 6:		Line 6:

	= Examples =		= Examples =
			<syntaxhighlight lang="bash">
	for each in `cat ../search\?q\=%7B%22source%22%3A%22legislation%22%2C%22congress%22%3A%22114%22%7D\&pageSize\=250 \| sed -n "s/.H\.R\.$[0-9]\{1,\}$./\1/p"`; do { wget https://www.congress.gov/bill/114th-congress/house-bill/$each; } done;		for each in `cat ../search\?q\=%7B%22source%22%3A%22legislation%22%2C%22congress%22%3A%22114%22%7D\&pageSize\=250 \| sed -n "s/.H\.R\.$[0-9]\{1,\}$./\1/p"`; do { wget https://www.congress.gov/bill/114th-congress/house-bill/$each; } done;
			</syntaxhighlight>

	= Resources =		= Resources =