mirror of https://github.com/tLDP/LDP
Finally fix/remove all broken URLs
This commit is contained in:
pbldp
parent
8810506461
commit
762403c691
|
|
@ -121,7 +121,7 @@ status inlined
|
|||
|
||||
\begin_layout Standard
|
||||
|
||||
<revision> <revnumber> 0.49.5.de.1</revnumber> <date>2006-08-21</date> <authorinitia
|
||||
<revision> <revnumber> 0.49.5.de.1</revnumber> <date>2006-08-23</date> <authorinitia
|
||||
ls>PB</authorinitials></revision>
|
||||
\end_layout
|
||||
|
||||
|
|
@ -9499,6 +9499,14 @@ Weitere Informationen
|
|||
Vorbereitung
|
||||
\end_layout
|
||||
|
||||
\begin_layout Standard
|
||||
|
||||
\lang english
|
||||
Dies ist nur notwendig, wenn der mitgelieferte Kernel und Netfilter nicht
|
||||
den Ansprüchen genügt und neue Featurs bereits verfügbar sind, jedoch noch
|
||||
nicht beinhaltet.
|
||||
\end_layout
|
||||
|
||||
\begin_layout Subsection
|
||||
Quellen besorgen
|
||||
\end_layout
|
||||
|
|
@ -9523,22 +9531,6 @@ Source tarball (f
|
|||
\end_inset
|
||||
|
||||
|
||||
\end_layout
|
||||
|
||||
\begin_layout Itemize
|
||||
Source RPM für die Neukompilierung des Programms (für Red Hat Systeme):
|
||||
|
||||
\begin_inset LatexCommand \url[ftp://ftp.redhat.com/redhat/linux/rawhide/SRPMS/SRPMS/]{ftp://ftp.redhat.com/redhat/linux/rawhide/SRPMS/SRPMS/}
|
||||
|
||||
\end_inset
|
||||
|
||||
oder ev.
|
||||
unter
|
||||
\begin_inset LatexCommand \url[http://www.netcore.fi/pekkas/linux/ipv6/]{http://www.netcore.fi/pekkas/linux/ipv6/}
|
||||
|
||||
\end_inset
|
||||
|
||||
|
||||
\end_layout
|
||||
|
||||
\begin_layout Subsection
|
||||
|
|
@ -12829,7 +12821,7 @@ A6, DNAME (WURDE ABGELEHNT!) und reverse IP6.ARPA: beschrieben in
|
|||
|
||||
sowie seit BIND 9 in Verwendung.
|
||||
Informationen zum aktuellen Stand sind unter
|
||||
\begin_inset LatexCommand \url[draft-ietf-dnsext-ipv6-addresses-00.txt]{http://www.ietf.org/internet-drafts/}
|
||||
\begin_inset LatexCommand \url[Domain Name System Extension (dnsext)]{http://www.ietf.org/ids.by.wg/dnsext.html}
|
||||
|
||||
\end_inset
|
||||
|
||||
|
|
@ -16970,14 +16962,6 @@ Gro
|
|||
Italien
|
||||
\end_layout
|
||||
|
||||
\begin_layout Itemize
|
||||
\begin_inset LatexCommand \url[Centro Studi e Laboratory Telecomunicazioni]{https://carmen.cselt.it/ipv6tb/}
|
||||
|
||||
\end_inset
|
||||
|
||||
|
||||
\end_layout
|
||||
|
||||
\begin_layout Itemize
|
||||
\begin_inset LatexCommand \url[Comv6]{http://www.comv6.com/}
|
||||
|
||||
|
|
@ -17003,7 +16987,7 @@ Japan
|
|||
\end_layout
|
||||
|
||||
\begin_layout Itemize
|
||||
\begin_inset LatexCommand \url[Internet Initiative Japan]{http://www.iij.ad.jp/IPv6/index-e.html}
|
||||
\begin_inset LatexCommand \url[Internet Initiative Japan]{http://www.iij.ad.jp/en/IPv6/}
|
||||
|
||||
\end_inset
|
||||
|
||||
|
|
@ -17227,14 +17211,8 @@ Anmerkung: Die folgenden Dienste sind meist nur mit einer g
|
|||
Net News (NNTP)
|
||||
\end_layout
|
||||
|
||||
\begin_layout Itemize
|
||||
|
||||
\lang english
|
||||
\begin_inset LatexCommand \url[news.ipv6.scarlet-internet.nl]{nntp://news.ipv6.scarlet-internet.nl/}
|
||||
|
||||
\end_inset
|
||||
|
||||
(erreichbar über alle SixXS POPs)
|
||||
\begin_layout Standard
|
||||
Fehlt etwas? Vorschläge sind Willkommen!
|
||||
\end_layout
|
||||
|
||||
\begin_layout Subsubsection
|
||||
|
|
@ -17846,7 +17824,7 @@ Englisch
|
|||
\size default
|
||||
\lang english
|
||||
,
|
||||
\begin_inset LatexCommand \url[Archive]{http://news.gmane.org/thread.php?group=gmane.network.ipv6.general}
|
||||
\begin_inset LatexCommand \url[Archive]{http://www.mail-archive.com/users@ipv6.org/}
|
||||
|
||||
\end_inset
|
||||
|
||||
|
|
|
|||
|
|
@ -569,8 +569,8 @@ Chinese
|
|||
|
||||
\begin_layout Standard
|
||||
A Chinese translation by Burma Chen <expns at yahoo dot com> (announced
|
||||
to me at 2002-10-31) can be found on the CLDP:
|
||||
\begin_inset LatexCommand \url[http://cldp.linuxforum.net/Linux-IPv6-HOWTO.html (broken)]{http://cldp.linuxforum.net/Linux-IPv6-HOWTO.html}
|
||||
to me at 2002-10-31) can be found on the TLDP:
|
||||
\begin_inset LatexCommand \url[http://www.ibiblio.org/pub/Linux/docs/HOWTO/translations/zh/Linux-IPv6-HOWTO.txt.gz]{http://www.ibiblio.org/pub/Linux/docs/HOWTO/translations/zh/Linux-IPv6-HOWTO.txt.gz}
|
||||
|
||||
\end_inset
|
||||
|
||||
|
|
@ -9239,6 +9239,11 @@ More information
|
|||
Preparation
|
||||
\end_layout
|
||||
|
||||
\begin_layout Standard
|
||||
This step is only needed if distributed kernel and netfilter doesn't fit
|
||||
your requirements and new features are available but still not built-in.
|
||||
\end_layout
|
||||
|
||||
\begin_layout Subsection
|
||||
Get sources
|
||||
\end_layout
|
||||
|
|
@ -9263,20 +9268,6 @@ Source tarball (for kernel patches):
|
|||
\end_inset
|
||||
|
||||
|
||||
\end_layout
|
||||
|
||||
\begin_layout Itemize
|
||||
Source RPM for rebuild of binary (for RedHat systems):
|
||||
\begin_inset LatexCommand \url[ftp://ftp.redhat.com/redhat/linux/rawhide/SRPMS/SRPMS/]{ftp://ftp.redhat.com/redhat/linux/rawhide/SRPMS/SRPMS/}
|
||||
|
||||
\end_inset
|
||||
|
||||
or perhaps also at
|
||||
\begin_inset LatexCommand \url[http://www.netcore.fi/pekkas/linux/ipv6/]{http://www.netcore.fi/pekkas/linux/ipv6/}
|
||||
|
||||
\end_inset
|
||||
|
||||
|
||||
\end_layout
|
||||
|
||||
\begin_layout Subsection
|
||||
|
|
@ -12108,7 +12099,7 @@ A6, DNAME (DEPRECATED NOW!) and reverse IP6.ARPA: specified in
|
|||
|
||||
, usable since BIND 9, but see also an information about the current state
|
||||
at
|
||||
\begin_inset LatexCommand \url[draft-ietf-dnsext-ipv6-addresses-00.txt]{http://www.ietf.org/internet-drafts/}
|
||||
\begin_inset LatexCommand \url[Domain Name System Extension (dnsext)]{http://www.ietf.org/ids.by.wg/dnsext.html}
|
||||
|
||||
\end_inset
|
||||
|
||||
|
|
@ -15905,14 +15896,6 @@ Germany
|
|||
Italy
|
||||
\end_layout
|
||||
|
||||
\begin_layout Itemize
|
||||
\begin_inset LatexCommand \url[Centro Studi e Laboratory Telecomunicazioni]{https://carmen.cselt.it/ipv6tb/}
|
||||
|
||||
\end_inset
|
||||
|
||||
|
||||
\end_layout
|
||||
|
||||
\begin_layout Itemize
|
||||
\begin_inset LatexCommand \url[Comv6]{http://www.comv6.com/}
|
||||
|
||||
|
|
@ -16182,12 +16165,8 @@ Note: These services are mostly only available with a valid IPv6 connection!
|
|||
Net News (NNTP)
|
||||
\end_layout
|
||||
|
||||
\begin_layout Itemize
|
||||
\begin_inset LatexCommand \url[news.ipv6.scarlet-internet.nl]{nntp://news.ipv6.scarlet-internet.nl/}
|
||||
|
||||
\end_inset
|
||||
|
||||
(accessible through all the SixXS POPs)
|
||||
\begin_layout Standard
|
||||
Something missing? Suggestions are welcome!
|
||||
\end_layout
|
||||
|
||||
\begin_layout Subsubsection
|
||||
|
|
@ -16715,7 +16694,7 @@ English
|
|||
\end_inset
|
||||
|
||||
,
|
||||
\begin_inset LatexCommand \url[Archive]{http://news.gmane.org/thread.php?group=gmane.network.ipv6.general}
|
||||
\begin_inset LatexCommand \url[Archive]{http://www.mail-archive.com/users@ipv6.org/}
|
||||
|
||||
\end_inset
|
||||
|
||||
|
|
@ -17139,7 +17118,7 @@ Releases 0.x
|
|||
\end_layout
|
||||
|
||||
\begin_layout Description
|
||||
0.49.5 2006-08-22/PB: fix/remove broken URLs
|
||||
0.49.5 2006-08-23/PB: fix/remove broken URLs
|
||||
\end_layout
|
||||
|
||||
\begin_layout Description
|
||||
|
|
|
|||
|
|
@ -8,7 +8,13 @@
|
|||
|
||||
use strict;
|
||||
use Net::HTTP;
|
||||
use Net::FTP;
|
||||
use Net::NNTP;
|
||||
use Crypt::SSLeay;
|
||||
use LWP::UserAgent;
|
||||
use XML::Dumper;
|
||||
use Socket;
|
||||
use Socket6;
|
||||
|
||||
my $debug = 0xffff & ~(0x20);
|
||||
|
||||
|
|
@ -19,7 +25,9 @@ my %hosts;
|
|||
|
||||
my $time = time;
|
||||
|
||||
my $dbfile = "url_database.xml";
|
||||
my $dbfile;
|
||||
|
||||
my $dbfile_suffix = ".url-database.xml";
|
||||
|
||||
|
||||
sub quote($) {
|
||||
|
|
@ -98,16 +106,56 @@ sub cleanup_old_urls() {
|
|||
};
|
||||
};
|
||||
|
||||
sub check_ipv6only($$) {
|
||||
print STDERR "DEBUG/check_ipv6only: begin\n" if ($debug & 0x10);
|
||||
print STDERR "DEBUG/check_ipv6only: check: " . $_[0] . "\n" if ($debug & 0x10);
|
||||
|
||||
my ($family, $socktype, $proto, $saddr, $canonname, @res);
|
||||
|
||||
@res = getaddrinfo($_[0], $_[1], AF_INET6, SOCK_STREAM);
|
||||
|
||||
if (scalar(@res) < 5) {
|
||||
return 1;
|
||||
};
|
||||
|
||||
my ($host, $port);
|
||||
$family = -1;
|
||||
|
||||
while (scalar(@res) >= 5) {
|
||||
($family, $socktype, $proto, $saddr, $canonname, @res) = @res;
|
||||
|
||||
($host, $port) = getnameinfo($saddr, NI_NUMERICHOST | NI_NUMERICSERV);
|
||||
|
||||
print STDERR "Trying to connect to $host port port $port...\n";
|
||||
|
||||
socket(Socket_Handle, $family, $socktype, $proto) || next;
|
||||
connect(Socket_Handle, $saddr) && last;
|
||||
|
||||
close(Socket_Handle);
|
||||
$family = -1;
|
||||
};
|
||||
|
||||
if ($family != -1) {
|
||||
print STDERR "connected to $host port $port\n";
|
||||
close(Socket_Handle);
|
||||
return 0;
|
||||
} else {
|
||||
warn "connect attempt failed\n";
|
||||
return 1;
|
||||
};
|
||||
};
|
||||
|
||||
sub check_urls() {
|
||||
print STDERR "DEBUG/check_urls: begin\n" if ($debug & 0x10);
|
||||
|
||||
for my $url (keys %$p_urls) {
|
||||
for my $url (sort keys %$p_urls) {
|
||||
if (defined $$p_urls{$url}->{'checktime'}) {
|
||||
if ($$p_urls{$url}->{'checktime'} > $time - 60*60*24*7) {
|
||||
# Checked during last 7 days - skip
|
||||
print STDERR "DEBUG/check_urls: checked during last 7 days - skip: $url\n" if ($debug & 0x10);
|
||||
next;
|
||||
if (defined $$p_urls{$url}->{'checkresult'} && $$p_urls{$url}->{'checkresult'} =~ /^ok/) {
|
||||
# Checked during last 7 days - skip
|
||||
print STDERR "DEBUG/check_urls: checked during last 7 days - skip: $url\n" if ($debug & 0x10);
|
||||
next;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
|
|
@ -130,16 +178,10 @@ sub check_urls() {
|
|||
$port = 80;
|
||||
} elsif ($proto eq "ftp") {
|
||||
$port = 21;
|
||||
$status = "skipped (ftp)";
|
||||
goto ("LABEL_END");
|
||||
} elsif ($proto eq "nntp") {
|
||||
$port = 119;
|
||||
$status = "skipped (nntp)";
|
||||
goto ("LABEL_END");
|
||||
} elsif ($proto eq "https") {
|
||||
$port = 443;
|
||||
$status = "skipped (https)";
|
||||
goto ("LABEL_END");
|
||||
};
|
||||
};
|
||||
|
||||
|
|
@ -151,28 +193,87 @@ sub check_urls() {
|
|||
goto ("LABEL_PRINT");
|
||||
};
|
||||
|
||||
# Check
|
||||
print STDERR "DEBUG/check_urls: open connection: $host:$port\n" if ($debug & 0x20);
|
||||
my $s = Net::HTTP->new(Host => $host, PeerPort => $port, Timeout => 30);
|
||||
if (! defined $s) {
|
||||
$status = "Host not found";
|
||||
goto ("LABEL_PRINT");
|
||||
};
|
||||
my $s;
|
||||
|
||||
print STDERR "DEBUG/check_urls: send HEAD request: $uri\n" if ($debug & 0x20);
|
||||
if ($s->write_request(HEAD => $uri, 'User-Agent' => "Mozilla/5.0") == 0) {
|
||||
$status = "trouble with uri";
|
||||
goto ("LABEL_PRINT");
|
||||
};
|
||||
if ($proto eq "ftp") {
|
||||
# Check FTP
|
||||
print STDERR "DEBUG/check_urls: open FTP connection: $host:$port\n" if ($debug & 0x20);
|
||||
$s = Net::FTP->new(Host => $host, Port => $port, Timeout => 30, Passive => 1);
|
||||
|
||||
print STDERR "DEBUG/check_urls: wait for response\n" if ($debug & 0x20);
|
||||
my($code, $mess, %h) = $s->read_response_headers;
|
||||
if (! defined $s) {
|
||||
$status = "Host not found";
|
||||
if (! check_ipv6only($host,$port)) {
|
||||
$status = "ok (IPv6 only)";
|
||||
};
|
||||
goto ("LABEL_PRINT");
|
||||
};
|
||||
|
||||
if (!$s->login("anonymous",'-anonymous@')) {
|
||||
$status = "FTP anonymous login failed";
|
||||
goto ("LABEL_PRINT");
|
||||
};
|
||||
|
||||
if (!$s->cwd($uri)) {
|
||||
$status = "FTP can't change to directory $uri";
|
||||
goto ("LABEL_PRINT");
|
||||
};
|
||||
|
||||
print STDERR "DEBUG/check_urls: check response\n" if ($debug & 0x10);
|
||||
if ($code !~ /^[23]/) {
|
||||
$status = "HTTP reports: $code";
|
||||
} else {
|
||||
$status = "ok";
|
||||
$s->quit;
|
||||
|
||||
} elsif ($proto eq "nntp") {
|
||||
my $s = Net::NNTP->new(Host => $host, Timeout => 30);
|
||||
|
||||
if (! defined $s) {
|
||||
$status = "Host not found";
|
||||
if (! check_ipv6only($host,$port)) {
|
||||
$status = "ok (IPv6 only)";
|
||||
};
|
||||
goto ("LABEL_PRINT");
|
||||
};
|
||||
$status = "ok";
|
||||
|
||||
$s->quit;
|
||||
|
||||
} elsif ($proto eq "https") {
|
||||
my $ua = new LWP::UserAgent;
|
||||
my $req = new HTTP::Request('HEAD', $url);
|
||||
my $res = $ua->request($req);
|
||||
|
||||
my $code = $res->code;
|
||||
|
||||
if ($code !~ /^[23]/) {
|
||||
$status = "HTTPS reports: $code";
|
||||
} else {
|
||||
$status = "ok";
|
||||
};
|
||||
} elsif ($proto eq "http") {
|
||||
# Check HTTP
|
||||
print STDERR "DEBUG/check_urls: open HTTP connection: $host:$port\n" if ($debug & 0x20);
|
||||
$s = Net::HTTP->new(Host => $host, PeerPort => $port, Timeout => 30);
|
||||
if (! defined $s) {
|
||||
$status = "Host not found";
|
||||
if (! check_ipv6only($host,$port)) {
|
||||
$status = "ok (IPv6 only)";
|
||||
};
|
||||
goto ("LABEL_PRINT");
|
||||
};
|
||||
|
||||
print STDERR "DEBUG/check_urls: send HEAD request: $uri\n" if ($debug & 0x20);
|
||||
if ($s->write_request(HEAD => $uri, 'User-Agent' => "Mozilla/5.0") == 0) {
|
||||
$status = "trouble with uri";
|
||||
goto ("LABEL_PRINT");
|
||||
};
|
||||
|
||||
print STDERR "DEBUG/check_urls: wait for response\n" if ($debug & 0x20);
|
||||
my($code, $mess, %h) = $s->read_response_headers;
|
||||
|
||||
print STDERR "DEBUG/check_urls: check response\n" if ($debug & 0x10);
|
||||
if ($code !~ /^[23]/) {
|
||||
$status = "HTTP reports: $code";
|
||||
} else {
|
||||
$status = "ok";
|
||||
};
|
||||
};
|
||||
LABEL_PRINT:
|
||||
if ($status ne "ok") {
|
||||
|
|
@ -192,7 +293,7 @@ sub report_urls() {
|
|||
print STDERR "DEBUG/report_urls: begin\n" if ($debug & 0x10);
|
||||
|
||||
for my $url (sort { $$p_urls{$a}->{'line'} <=> $$p_urls{$b}->{'line'} } ( keys %$p_urls)) {
|
||||
if ($$p_urls{$url}->{'checkresult'} eq "ok") {
|
||||
if ($$p_urls{$url}->{'checkresult'} =~ /^ok/) {
|
||||
next;
|
||||
};
|
||||
|
||||
|
|
@ -209,6 +310,16 @@ sub report_urls() {
|
|||
|
||||
##### Main
|
||||
|
||||
if (! defined $ARGV[0] || $ARGV[0] eq "") {
|
||||
die "Missing file name (arg1)";
|
||||
};
|
||||
|
||||
if (! -f $ARGV[0]) {
|
||||
die "Argument 1 is not an existing file: " . $ARGV[0];
|
||||
};
|
||||
|
||||
$dbfile = $ARGV[0] . $dbfile_suffix;
|
||||
|
||||
load_urls();
|
||||
extract_urls($ARGV[0]);
|
||||
cleanup_old_urls();
|
||||
|
|
|
|||
Loading…
Reference in New Issue