anduin / apt-mirror

#!/usr/bin/perl =pod =head1 NAME apt-mirror - apt sources mirroring tool =head1 SYNOPSIS apt-mirror [configfile] =head1 DESCRIPTION A small and efficient tool that lets you mirror a part of or the whole Debian GNU/Linux distribution or any other apt sources. Main features: * It uses a config similar to APT's F<sources.list> * It's fully pool compliant * It supports multithreaded downloading * It supports multiple architectures at the same time * It can automatically remove unneeded files * It works well on an overloaded Internet connection * It never produces an inconsistent mirror including while mirroring * It works on all POSIX compliant systems with Perl and wget =head1 COMMENTS apt-mirror uses F</etc/apt/mirror.list> as a configuration file. By default it is tuned to official Debian or Ubuntu mirrors. Change it for your needs. After you setup the configuration file you may run as root: # su - apt-mirror -c apt-mirror Or uncomment the line in F</etc/cron.d/apt-mirror> to enable daily mirror updates. =head1 FILES F</etc/apt/mirror.list> Main configuration file F</etc/cron.d/apt-mirror> Cron configuration template F</var/spool/apt-mirror/mirror> Mirror places here F</var/spool/apt-mirror/skel> Place for temporarily downloaded indexes F</var/spool/apt-mirror/var> Log files placed here. URLs and MD5 checksums also here. =head1 CONFIGURATION EXAMPLES The mirror.list configuration supports many options, the file is well commented explaining each option. Here are some sample mirror configuration lines showing the various supported ways: Normal: deb http://example.com/debian stable main contrib non-free Arch Specific: (many other architectures are supported) deb-powerpc http://example.com/debian stable main contrib non-free HTTP and FTP Auth or non-standard port: deb http://user:pass@example.com:8080/debian stable main contrib non-free HTTPS with sending Basic HTTP authentication information (plaintext username and password) for all requests: (this was default behaviour of Wget 1.10.2 and prior and is needed for some servers with new version of Wget) set auth_no_challenge 1 deb https://user:pass@example.com:443/debian stable main contrib non-free HTTPS without checking certificate: set no_check_certificate 1 deb https://example.com:443/debian stable main contrib non-free Source Mirroring: deb-src http://example.com/debian stable main contrib non-free =head1 AUTHORS Dmitry N. Hramtsov E<lt>hdn@nsu.ruE<gt> Brandon Holtsclaw E<lt>me@brandonholtsclaw.comE<gt> =cut use warnings; use strict; use File::Copy; use File::Compare; use File::Path qw(make_path); use File::Basename; use Fcntl qw(:flock); my $config_file; my %config_variables = ( "defaultarch" => `dpkg --print-architecture 2>/dev/null` || 'i386', "nthreads" => 20, "base_path" => '/var/spool/apt-mirror', "mirror_path" => '$base_path/mirror', "skel_path" => '$base_path/skel', "var_path" => '$base_path/var', "cleanscript" => '$var_path/clean.sh', "_contents" => 1, "_autoclean" => 0, "_tilde" => 0, "_plus" => 0, "limit_rate" => '100m', "run_postmirror" => 1, "auth_no_challenge" => 0, "no_check_certificate" => 0, "unlink" => 0, "paranoid" => 0, "postmirror_script" => '$var_path/postmirror.sh', "use_proxy" => 'off', "http_proxy" => '', "https_proxy" => '', "proxy_user" => '', "proxy_password" => '' ); my @config_binaries = (); my @config_sources = (); my @release_urls; my @index_urls; my @childrens = (); my %skipclean = (); my %clean_directory = (); my @hash_strength = qw(SHA512 SHA256 SHA1 MD5Sum); my %packages_hashes = ( SHA512 => "SHA512", SHA256 => "SHA256", SHA1 => "SHA1", MD5Sum => "MD5sum", ); my %sources_hashes = ( SHA512 => "Checksums-Sha512", SHA256 => "Checksums-Sha256", SHA1 => "Checksums-Sha1", MD5Sum => "Files", ); my %verify_commands = ( SHA512 => "sha512sum", SHA256 => "sha256sum", SHA1 => "sha1sum", MD5Sum => "md5sum", ); my %checksum_filenames = ( SHA512 => "SHA512", SHA256 => "SHA256", SHA1 => "SHA1", MD5Sum => "MD5", ); # Mapping of files downloaded from a by-hash directory to their canonical locations. my %hashsum_to_files = (); # Mapping of all the checksums for a given canonical filename. my %file_to_hashsums; my %urls_checksums = (); ###################################################################################### ## Setting up $config_file variable $config_file = "/etc/apt/mirror.list"; # Default value if ( $_ = shift ) { die("apt-mirror: invalid config file specified") unless -e $_; $config_file = $_; } chomp $config_variables{"defaultarch"}; ###################################################################################### ## Common subroutines sub round_number { my $n = shift; my $minus = $n < 0 ? '-' : ''; $n = abs($n); $n = int( ( $n + .05 ) * 10 ) / 10; $n .= '.0' unless $n =~ /\./; $n .= '0' if substr( $n, ( length($n) - 1 ), 1 ) eq '.'; chop $n if $n =~ /\.\d\d0$/; return "$minus$n"; } sub format_bytes { my $bytes = shift; my $bytes_out = '0'; my $size_name = 'bytes'; my $KiB = 1024; my $MiB = 1024 * 1024; my $GiB = 1024 * 1024 * 1024; if ( $bytes >= $KiB ) { $bytes_out = $bytes / $KiB; $size_name = 'KiB'; if ( $bytes >= $MiB ) { $bytes_out = $bytes / $MiB; $size_name = 'MiB'; if ( $bytes >= $GiB ) { $bytes_out = $bytes / $GiB; $size_name = 'GiB'; } } $bytes_out = round_number($bytes_out); } else { $bytes_out = $bytes; $size_name = 'bytes'; } return "$bytes_out $size_name"; } sub get_variable { my $value = $config_variables{ shift @_ }; my $count = 16; while ( $value =~ s/\$(\w+)/$config_variables{$1}/xg ) { die("apt-mirror: too many substitution while evaluating variable") if ( $count-- ) < 0; } return $value; } sub quoted_path { my $path = shift; $path =~ s/'/'\\''/g; return "'" . $path . "'"; } sub lock_aptmirror { open( LOCK_FILE, '>', get_variable("var_path") . "/apt-mirror.lock" ); my $lock = flock( LOCK_FILE, LOCK_EX | LOCK_NB ); if ( !$lock ) { die("apt-mirror is already running, exiting"); } } sub unlock_aptmirror { close(LOCK_FILE); unlink( get_variable("var_path") . "/apt-mirror.lock" ); } sub delete_corrupted_files { my $stage = shift; my $found = 0; foreach my $hash (@hash_strength) { my $file = get_variable("var_path") . "/${stage}-${hash}"; if (-s $file) { my $pipe; open $pipe, "-|", qq(env LC_ALL=C ${verify_commands{$hash}} --check --quiet ${file} 2>/dev/null) or die "Cannot run ${verify_commands{$hash}}"; while (<$pipe>) { my ($filename) = /^(.*): FAILED/; if (-f $filename) { $found++; print "$filename is corrupted, deleting....\n"; unlink $filename or die "Cannot delete $filename."; } } close $pipe; } } return $found; } sub download_urls { my $stage = shift; my @urls; my $i = 0; my $pid; my $nthreads = get_variable("nthreads"); my @args = (); local $| = 1; @urls = @_; $nthreads = @urls if @urls < $nthreads; if ( get_variable("auth_no_challenge") == 1 ) { push( @args, "--auth-no-challenge" ); } if ( get_variable("no_check_certificate") == 1 ) { push( @args, "--no-check-certificate" ); } if ( get_variable("unlink") == 1 ) { push( @args, "--unlink" ); } if ( length( get_variable("use_proxy") ) && ( get_variable("use_proxy") eq 'yes' || get_variable("use_proxy") eq 'on' ) ) { if ( length( get_variable("http_proxy") ) || length( get_variable("https_proxy") ) ) { push( @args, "-e use_proxy=yes" ); } if ( length( get_variable("http_proxy") ) ) { push( @args, "-e http_proxy=" . get_variable("http_proxy") ); } if ( length( get_variable("https_proxy") ) ) { push( @args, "-e https_proxy=" . get_variable("https_proxy") ); } if ( length( get_variable("proxy_user") ) ) { push( @args, "-e proxy_user=" . get_variable("proxy_user") ); } if ( length( get_variable("proxy_password") ) ) { push( @args, "-e proxy_password=" . get_variable("proxy_password") ); } } print "Downloading " . scalar(@urls) . " $stage files using $nthreads threads...\n"; if (get_variable("paranoid")) { my %fh = (); foreach my $hash (@hash_strength) { open $fh{$hash}, ">", get_variable("var_path") . "/${stage}-${hash}" or die ("apt-mirror: Cannot write to ${stage}-${hash}"); } foreach (@urls) { if ($urls_checksums{$_}) { my ($hash, $hashsum) = @{$urls_checksums{$_}}; my $fh = $fh{$hash}; print $fh $hashsum . " " . sanitise_uri($_) . "\n"; } } foreach my $hash (@hash_strength) { close $fh{$hash}; } } my @url_fds; for ($i=0; $i<$nthreads; $i++) { open ( $url_fds[$i], ">", get_variable("var_path") . "/$stage-urls.$i") or die("apt-mirror: can't write to intermediate file ($stage-urls.$i)"); } for ($i=scalar(@urls)-1; $i>=0; $i--) { my $thread = $i % $nthreads; print { $url_fds[$thread] } $urls[$i] . "\n"; } foreach (@url_fds) { close $_ or die("apt-mirror: can't close intermediate file ($stage-urls.$i)"); } for ($i=0; $i<$nthreads; $i++) { $pid = fork(); die("apt-mirror: can't do fork in download_urls") if !defined($pid); if ( $pid == 0 ) { exec 'wget', '--no-if-modified-since', '--no-cache', '--limit-rate=' . get_variable("limit_rate"), '-T', '60', '-t', '1', '-r', '-N', '-l', 'inf', '-o', get_variable("var_path") . "/$stage-log.$i", '-i', get_variable("var_path") . "/$stage-urls.$i", @args; # shouldn't reach this unless exec fails die("\n\nCould not run wget, please make sure its installed and in your path\n\n"); } push @childrens, $pid; } print "Begin time: " . localtime() . "\n[" . scalar(@childrens) . "]... "; while ( scalar @childrens ) { my $dead = wait(); @childrens = grep { $_ != $dead } @childrens; print "[" . scalar(@childrens) . "]... "; } print "\nEnd time: " . localtime() . "\n\n"; if (get_variable("paranoid")) { if (delete_corrupted_files($stage) > 0) { die "Some files were corrupted while downloading, aborting..."; } } if (scalar keys %hashsum_to_files > 0) { foreach my $hashsum_filename (keys %hashsum_to_files) { foreach my $filename (@{$hashsum_to_files{$hashsum_filename}}) { copy_file( $hashsum_filename, $filename ); } } } } ## Parse config sub parse_config_line { my $pattern_deb_line = qr/^[\t ]*(?<type>deb-src|deb)(?:-(?<arch>[\w\-]+))?[\t ]+(?:\[(?<options>[^\]]+)\][\t ]+)?(?<uri>[^\s]+)[\t ]+(?<components>.+)$/; my $line = $_; my %config; if ( $line =~ $pattern_deb_line ) { $config{'type'} = $+{type}; $config{'arch'} = $+{arch}; $config{'options'} = $+{options} ? $+{options} : ""; $config{'uri'} = $+{uri}; $config{'components'} = $+{components}; if ( $config{'options'} =~ /arch=((?<arch>[\w\-]+)[,]*)/g ) { $config{'arch'} = $+{arch}; } $config{'components'} = [ split /\s+/, $config{'components'} ]; } elsif ( $line =~ /set[\t ]+(?<key>[^\s]+)[\t ]+(?<value>"[^"]+"|'[^']+'|[^\s]+)/ ) { $config{'type'} = 'set'; $config{'key'} = $+{key}; $config{'value'} = $+{value}; $config{'value'} =~ s/^'(.*)'$/$1/; $config{'value'} =~ s/^"(.*)"$/$1/; } elsif ( $line =~ /(?<type>clean|skip-clean)[\t ]+(?<uri>[^\s]+)/ ) { $config{'type'} = $+{type}; $config{'uri'} = $+{uri}; } return %config; } sub sanitise_uri { my $uri = shift; $uri =~ s[^(\w+)://][]; $uri =~ s/^([^@]+)?@?// if (split '/',$uri)[0] =~ /@/; $uri =~ s/~/\%7E/g if get_variable("_tilde"); $uri =~ s/\+/\%2B/g if get_variable("_plus"); $uri =~ s[/$][]; return $uri; } open CONFIG, "<$config_file" or die("apt-mirror: can't open config file ($config_file)"); while (<CONFIG>) { next if /^\s*#/; next unless /\S/; my $line = $_; my %config_line = parse_config_line; if ( $config_line{'type'} eq "set" ) { $config_variables{ $config_line{'key'} } = $config_line{'value'}; next; } elsif ( $config_line{'type'} eq "deb" ) { my $arch = $config_line{'arch'}; $arch = get_variable("defaultarch") if ! defined $config_line{'arch'}; push @config_binaries, [ $arch, $config_line{'uri'}, @{$config_line{'components'}} ]; next; } elsif ( $config_line{'type'} eq "deb-src" ) { push @config_sources, [ $config_line{'uri'}, @{$config_line{'components'}} ]; next; } elsif ( $config_line{'type'} =~ /(skip-clean|clean)/ ) { my $link = sanitise_uri($config_line{'uri'}); if ( $config_line{'type'} eq "skip-clean" ) { $skipclean{ $link } = 1; } elsif ( $config_line{'type'} eq "clean" ) { $clean_directory{ $link } = 1; } next; } die("apt-mirror: invalid line in config file ($.: $line ...)"); } close CONFIG; die("Please explicitly specify 'defaultarch' in mirror.list") unless get_variable("defaultarch"); ###################################################################################### ## Create the 3 needed directories if they don't exist yet my @needed_directories = ( get_variable("mirror_path"), get_variable("skel_path"), get_variable("var_path") ); foreach my $needed_directory (@needed_directories) { unless ( -d $needed_directory ) { make_path($needed_directory) or die("apt-mirror: can't create $needed_directory directory"); } } # ####################################################################################### lock_aptmirror(); ###################################################################################### ## Skel download my %urls_to_download = (); my ( $url, $arch ); sub remove_double_slashes { local $_ = shift; while (s[/\./][/]g) { } while (s[(?<!:)//][/]g) { } while (s[(?<!:/)/[^/]+/\.\./][/]g) { } s/~/\%7E/g if get_variable("_tilde"); s/\+/\%2B/g if get_variable("_plus"); return $_; } sub add_url_to_download { my $url = remove_double_slashes(shift); my $size = shift; my $strongest_hash = shift; my $hash = shift; my $hashsum = shift; my $acquire_by_hash = shift; my $canonical_filename = sanitise_uri($url); $skipclean{$canonical_filename} = 1; if ($acquire_by_hash) { # If the optional hashsum was passed as an argument # - download the strongest hash only # - make a copy to the canonical location # - make a copy for the other known hash versions $url = dirname($url) . "/by-hash/${hash}/${hashsum}"; my $hashsum_filename = dirname($canonical_filename) . "/by-hash/${hash}/${hashsum}"; $skipclean{$hashsum_filename} = 1; if ($hash eq $strongest_hash) { # This is the strongest hash, which is the one to download. # Also need to remember to which canonical location it should be linked. $hashsum_to_files{$hashsum_filename} ||= []; push @{$hashsum_to_files{$hashsum_filename}}, $canonical_filename; $urls_to_download{$url} = $size; $urls_checksums{$url} = [ $hash, $hashsum ]; } else { # We are not going to download using this checksum, but we still # need to know where to put the checksum. $file_to_hashsums{$canonical_filename} ||= []; push @{$file_to_hashsums{$canonical_filename}}, $hashsum_filename; } } else { # Not using by-hash, so download the file only. $urls_to_download{$url} = $size; if ($strongest_hash and ($hash eq $strongest_hash)) { $urls_checksums{$url} = [ $hash, $hashsum ]; } } } foreach (@config_sources) { my ( $uri, $distribution, @components ) = @{$_}; if (@components) { $url = $uri . "/dists/" . $distribution . "/"; } else { $url = $uri . "/" . $distribution . "/"; } add_url_to_download( $url . "InRelease" ); add_url_to_download( $url . "Release" ); add_url_to_download( $url . "Release.gpg" ); } foreach (@config_binaries) { my ( $arch, $uri, $distribution, @components ) = @{$_}; if (@components) { $url = $uri . "/dists/" . $distribution . "/"; } else { $url = $uri . "/" . $distribution . "/"; } add_url_to_download( $url . "InRelease" ); add_url_to_download( $url . "Release" ); add_url_to_download( $url . "Release.gpg" ); } chdir get_variable("skel_path") or die("apt-mirror: can't chdir to skel"); @release_urls = sort keys %urls_to_download; download_urls( "release", @release_urls ); ###################################################################################### ## Download all relevant metadata %urls_to_download = (); sub find_metadata_in_release { # Look in the Release file for any files we need to download my ( $arch, $uri, $distribution, @components ) = @_; my ( $release_uri, $release_path, $line ) = ''; my $component_regex = undef; my $arch_regex = "(?:${arch}|all)"; my $compressed_extension_regex = '(?:\.(?:gz|bz2|xz|lzma))$'; my $dist_uri; my $hash_type_regex = "(?:" . join("|", @hash_strength) . ")"; if (@components) { $dist_uri = remove_double_slashes($uri . "/dists/" . $distribution . "/"); $component_regex = "(?:" . join("|", @components) . ")"; } else { $dist_uri = remove_double_slashes($uri . "/" . $distribution . "/"); } my $stream; foreach my $release_filename ("InRelease", "Release") { $release_uri = $dist_uri . $release_filename; $release_path = get_variable("skel_path") . "/" . sanitise_uri($release_uri); last if ( open $stream, "<", $release_path); $stream = undef; } unless ( $stream ) { warn( "Failed to find InRelease or Release in " . get_variable("skel_path") . "/" . sanitise_uri($dist_uri) ); return 0; } my $hash = undef; my %avaiable_hashes = (); my $acquire_by_hash = 0; my @parts_to_download = (); while ( $line = <$stream> ) { chomp $line; if ($hash) { if ( $line =~ /^ +(.*)$/ ) { my @parts = split( / +/, $1 ); if ( @parts == 3 ) { my ( $hashsum, $size, $filename ) = @parts; push @parts, $hash; if ($arch eq "source") { if ($component_regex) { # Debian repository format https://wiki.debian.org/DebianRepository/Format#Debian_Repository_Format if ( ( $filename =~ m{^${component_regex}/source/Sources${compressed_extension_regex}} ) or ( $filename =~ m{^${component_regex}/Contents-source${compressed_extension_regex}} ) ) { push @parts_to_download, \@parts; } } else { # Flat repository format https://wiki.debian.org/DebianRepository/Format#Flat_Repository_Format if ($filename =~ m{^Sources${compressed_extension_regex}} ) { push @parts_to_download, \@parts; } } } else { if ($component_regex) { # Debian repository format https://wiki.debian.org/DebianRepository/Format#Debian_Repository_Format if ( ( $filename =~ m{^${component_regex}/Contents-${arch_regex}${compressed_extension_regex}} ) or ( $filename =~ m{^Contents-${arch_regex}${compressed_extension_regex}} ) or ( $filename =~ m{^Packages${compressed_extension_regex}} ) or ( $filename =~ m{^${component_regex}/binary-${arch_regex}/Packages${compressed_extension_regex}} ) or ( $filename =~ m{^${component_regex}/binary-${arch_regex}/Release$} ) or ( $filename =~ m{^${component_regex}/cnf/Commands-${arch_regex}${compressed_extension_regex}} ) or ( $filename =~ m{^${component_regex}/dep11/Components-${arch_regex}.*${compressed_extension_regex}} ) or ( $filename =~ m{^${component_regex}/dep11/icons-.*${compressed_extension_regex}} ) or ( $filename =~ m{^${component_regex}/i18n/Translation-.*${compressed_extension_regex}} ) ) { push @parts_to_download, \@parts; } } else { # Flat repository format https://wiki.debian.org/DebianRepository/Format#Flat_Repository_Format if ($filename =~ m{^Packages${compressed_extension_regex}}) { push @parts_to_download, \@parts; } } } } else { warn("Malformed checksum line \"$1\" in $release_uri"); } } else { $hash = undef; } } if ( not $hash ) { if ( $line =~ /^(${hash_type_regex}):$/ ) { $hash = $1; $avaiable_hashes{$hash} = 1; } elsif ( $line eq "Acquire-By-Hash: yes" ) { $acquire_by_hash = 1; } } } close $stream; my $strongest_hash; if ($acquire_by_hash) { foreach (@hash_strength) { if ($avaiable_hashes{$_}) { $strongest_hash = $_; last; } } unless ($strongest_hash) { warn("Cannot find a supported hash in $release_uri, will download from canonical locations."); $acquire_by_hash = 0; } } foreach (@parts_to_download) { my ( $hashsum, $size, $filename, $hash ) = @{$_}; if ($acquire_by_hash) { add_url_to_download( $dist_uri . $filename, $size, $strongest_hash, $hash, $hashsum, 1 ); } else { add_url_to_download( $dist_uri . $filename, $size, $strongest_hash, $hash, $hashsum, 0 ); } } return 1; } print "Processing metadata files from releases ["; foreach (@config_binaries) { my ( $arch, $uri, $distribution, @components ) = @{$_}; print "M"; unless (find_metadata_in_release( $arch, $uri, $distribution, @components)) { # Insecure repo with no release file - try to get the well known indices foreach my $file_extension (".gz", ".bz2", ".xz", ".lzma", "") { if (@components) { # Debian repo foreach my $component (@components) { foreach my $path ( "/dists/${distribution}/${component}/binary-${arch}/Packages", "/dists/${distribution}/${component}/binary-all/Packages", "/dists/${distribution}/${component}/Contents-${arch}", "/dists/${distribution}/${component}/Contents-all", "/dists/${distribution}/Contents-${arch}", "/dists/${distribution}/Contents-all", ) { add_url_to_download( "${uri}/${path}${file_extension}" ); } } } else { # Flat repo foreach my $path ( "${distribution}/Packages", "${distribution}/Contents-${arch}", "${distribution}/Contents-all", ) { add_url_to_download( "${uri}/${path}${file_extension}" ); } } } } } foreach (@config_sources) { my ( $uri, $distribution, @components ) = @{$_}; print "M"; unless (find_metadata_in_release( "source", $uri, $distribution, @components)) { # Insecure repo with no release file - try to get the well known indices foreach my $file_extension (".gz", ".bz2", ".xz", ".lzma", "") { if (@components) { # Debian repo foreach my $path ( "${distribution}/source/Sources", "${distribution}/Contents-source", ) { add_url_to_download( "${uri}/${path}${file_extension}" ); } } else { # Flat repo add_url_to_download( "${uri}/${distribution}/Sources${file_extension}" ); } } } } print "]\n\n"; @index_urls = sort keys %urls_to_download; download_urls( "index", @index_urls ); ###################################################################################### ## Main download preparations %urls_to_download = (); my %files_fh; open $files_fh{ALL}, ">" . get_variable("var_path") . "/ALL" or die("apt-mirror: can't write to intermediate file (ALL)"); open $files_fh{NEW}, ">" . get_variable("var_path") . "/NEW" or die("apt-mirror: can't write to intermediate file (NEW)"); foreach my $hash (@hash_strength) { open $files_fh{$hash}, ">" . get_variable("var_path") . "/" . ${checksum_filenames{$hash}} or die("apt-mirror: can't write to intermediate file (${hash})"); } my %stat_cache = (); sub _stat { my ($filename) = shift; return @{ $stat_cache{$filename} } if exists $stat_cache{$filename}; my @res = stat($filename); $stat_cache{$filename} = \@res; return @res; } sub clear_stat_cache { %stat_cache = (); } sub need_update { my $filename = shift; my $size_on_server = shift; my ( undef, undef, undef, undef, undef, undef, undef, $size ) = _stat($filename); return 1 unless ($size); return 0 if $size_on_server == $size; if ( get_variable("unlink") == 1 ) { unlink $filename; } return 1; } sub process_index { my $uri = shift; my $index = shift; my $optional = shift; my ( $path, $package, $mirror, $files ) = ''; $path = sanitise_uri($uri); local $/ = "\n\n"; $mirror = get_variable("mirror_path") . "/" . $path; if (-e "$path/$index.gz" ) { system("gunzip < $path/$index.gz > $path/$index"); } elsif (-e "$path/$index.xz" ) { system("xz -d < $path/$index.xz > $path/$index"); } elsif (-e "$path/$index.lzma" ) { system("xz -d < $path/$index.xz > $path/$index"); } elsif (-e "$path/$index.bz2" ) { system("bzip2 -d < $path/$index.bz2 > $path/$index"); } unless ( open STREAM, "<$path/$index" ) { if ($optional) { return; } warn("apt-mirror: can't open index $path/$index in process_index"); return; } while ( $package = <STREAM> ) { local $/ = "\n"; chomp $package; my ( undef, %lines ) = split( /^([\w\-]+): */m, $package ); chomp(%lines); if ( exists $lines{"Filename"} ) { # Packages index my $filename = remove_double_slashes( $path . "/" . $lines{"Filename"}); $skipclean{ $filename } = 1; print { $files_fh{ALL} } $filename . "\n"; foreach my $hash (@hash_strength) { my $index_hash = $packages_hashes{$hash}; print { $files_fh{$hash} } $lines{$index_hash} . " " . $filename . "\n" if $lines{$index_hash}; } if ( need_update( $mirror . "/" . $lines{"Filename"}, $lines{"Size"} ) ) { my $hashsum = undef; my $hash = undef; foreach $hash (@hash_strength) { my $index_hash = $packages_hashes{$hash}; if ($lines{$index_hash}) { $hashsum = ${lines{$index_hash}}; last; } } print { $files_fh{NEW} } $filename. "\n"; add_url_to_download( $uri . "/" . $lines{"Filename"}, $lines{"Size"}, $hash, $hash, $hashsum, 0 ); } } else { # Sources index $lines{"Directory"} = "" unless defined $lines{"Directory"}; foreach my $hash (@hash_strength) { my $index_hash = $sources_hashes{$hash}; if ($lines{$index_hash}) { foreach ( split( /\n/, $lines{$index_hash} ) ) { next if $_ eq ''; my @file = split; die("apt-mirror: invalid Sources format") if @file != 3; my $download_url = $uri . "/" . $lines{"Directory"} . "/" . $file[2]; my $filename = remove_double_slashes( $path . "/" . $lines{"Directory"} . "/" . $file[2] ); print { $files_fh{$hash} } $file[0] . " " . ${filename} . "\n"; unless ($skipclean{ $filename }) { $skipclean{ $filename } = 1; print { $files_fh{ALL} } ${filename} . "\n"; if ( need_update( $mirror . "/" . $lines{"Directory"} . "/" . $file[2], $file[1] ) ) { print { $files_fh{NEW} } ${download_url} . "\n"; add_url_to_download( $uri . "/" . $lines{"Directory"} . "/" . $file[2], $file[1], $hash, $hash, $file[0], 0 ); } } } } } } } close STREAM; } print "Processing indexes: ["; foreach (@config_sources) { my ( $uri, $distribution, @components ) = @{$_}; print "S"; if (@components) { my $component; foreach $component (@components) { process_index( $uri, "/dists/$distribution/$component/source/Sources" ); } } else { process_index( $uri, "/$distribution/Sources" ); } } foreach (@config_binaries) { my ( $arch, $uri, $distribution, @components ) = @{$_}; print "P"; if (@components) { my $component; foreach $component (@components) { process_index( $uri, "/dists/$distribution/$component/binary-$arch/Packages" ); process_index( $uri, "/dists/$distribution/$component/binary-all/Packages", 1 ); } } else { process_index( $uri, "/$distribution/Packages" ); } } clear_stat_cache(); print "]\n\n"; foreach my $fh (values %files_fh) { close $fh; } ###################################################################################### ## Main download chdir get_variable("mirror_path") or die("apt-mirror: can't chdir to mirror"); my $need_bytes = 0; foreach ( values %urls_to_download ) { $need_bytes += $_; } my $size_output = format_bytes($need_bytes); print "$size_output will be downloaded into archive.\n"; download_urls( "archive", sort keys %urls_to_download ); ###################################################################################### ## Copy skel to main archive sub copy_file { my ( $from, $to ) = @_; my $dir = dirname($to); return unless -f $from; make_path($dir) unless -d $dir; if ( get_variable("unlink") == 1 ) { if ( compare( $from, $to ) != 0 ) { unlink($to); } } my @stat_from = stat($from); if ( -f $to ) { my @stat_to = stat($to); return if ("@stat_to" eq "@stat_from"); } unless ( link( $from, $to ) or copy( $from, $to ) ) { warn("apt-mirror: can't copy $from to $to"); return; } my ( $atime, $mtime ) = @stat_from[ 8, 9 ]; utime( $atime, $mtime, $to ) or die("apt-mirror: can't utime $to"); } foreach (@release_urls, @index_urls) { die("apt-mirror: invalid url in index_urls") unless s[^(\w+)://][]; copy_file( get_variable("skel_path") . "/" . sanitise_uri("$_"), get_variable("mirror_path") . "/" . sanitise_uri("$_") ); my $sanitized_uri = sanitise_uri($_); # If we downloaded any files from a checksum location, now is the time to # populate the canonical filename. if ($hashsum_to_files{$sanitized_uri}) { foreach my $filename (@{$hashsum_to_files{$sanitized_uri}}) { copy_file( get_variable("mirror_path") . "/" . $sanitized_uri, get_variable("mirror_path") . "/" . $filename ); if ($file_to_hashsums{$filename}) { foreach my $hashsum_filename (@{$file_to_hashsums{$filename}}) { copy_file( get_variable("mirror_path") . "/" . $sanitized_uri, get_variable("mirror_path") . "/" . $hashsum_filename ); } } } } } ###################################################################################### ## Make cleaning script my ( @rm_dirs, @rm_files ) = (); my $unnecessary_bytes = 0; sub process_symlink { return 1; # symlinks are always needed } sub process_file { my $file = shift; $file =~ s[~][%7E]g if get_variable("_tilde"); $file =~ s[\+][%2B]g if get_variable("_plus"); return 1 if $skipclean{$file}; push @rm_files, sanitise_uri($file); my ( undef, undef, undef, undef, undef, undef, undef, $size, undef, undef, undef, undef, $blocks ) = stat($file); $unnecessary_bytes += $blocks * 512; return 0; } sub process_directory { my $dir = shift; my $is_needed = 0; return 1 if $skipclean{$dir}; opendir( my $dir_h, $dir ) or die "apt-mirror: can't opendir $dir: $!"; foreach ( grep { !/^\.$/ && !/^\.\.$/ } readdir($dir_h) ) { my $item = $dir . "/" . $_; $is_needed |= process_directory($item) if -d $item && !-l $item; $is_needed |= process_file($item) if -f $item; $is_needed |= process_symlink($item) if -l $item; } closedir $dir_h; push @rm_dirs, $dir unless $is_needed; return $is_needed; } chdir get_variable("mirror_path") or die("apt-mirror: can't chdir to mirror"); foreach ( keys %clean_directory ) { process_directory($_) if -d $_ && !-l $_; } open CLEAN, ">" . get_variable("cleanscript") or die("apt-mirror: can't open clean script file"); my ( $i, $total ) = ( 0, scalar @rm_files ); if ( get_variable("_autoclean") ) { my $size_output = format_bytes($unnecessary_bytes); print "$size_output in $total files and " . scalar(@rm_dirs) . " directories will be freed..."; chdir get_variable("mirror_path") or die("apt-mirror: can't chdir to mirror"); foreach (@rm_files) { unlink $_; } foreach (@rm_dirs) { rmdir $_; } } else { my $size_output = format_bytes($unnecessary_bytes); print "$size_output in $total files and " . scalar(@rm_dirs) . " directories can be freed.\n"; print "Run " . get_variable("cleanscript") . " for this purpose.\n\n"; print CLEAN "#!/bin/sh\n"; print CLEAN "set -e\n\n"; print CLEAN "cd " . quoted_path(get_variable("mirror_path")) . "\n\n"; print CLEAN "echo 'Removing $total unnecessary files [$size_output]...'\n"; foreach (@rm_files) { print CLEAN "rm -f '$_'\n"; print CLEAN "echo -n '[" . int( 100 * $i / $total ) . "\%]'\n" unless $i % 500; print CLEAN "echo -n .\n" unless $i % 10; $i++; } print CLEAN "echo 'done.'\n"; print CLEAN "echo\n\n"; $i = 0; $total = scalar @rm_dirs; print CLEAN "echo 'Removing $total unnecessary directories...'\n"; foreach (@rm_dirs) { print CLEAN "if test -d '$_'; then rm -fr '$_'; fi\n"; print CLEAN "echo -n '[" . int( 100 * $i / $total ) . "\%]'\n" unless $i % 50; print CLEAN "echo -n .\n"; $i++; } print CLEAN "echo 'done.'\n"; print CLEAN "echo\n"; close CLEAN; } # Make clean script executable my $perm = ( stat get_variable("cleanscript") )[2] & 07777; chmod( $perm | 0111, get_variable("cleanscript") ); if ( get_variable("run_postmirror") ) { print "Running the Post Mirror script ...\n"; print "(" . get_variable("postmirror_script") . ")\n\n"; if ( -x get_variable("postmirror_script") ) { system( get_variable("postmirror_script"), '' ); } else { system( '/bin/sh', get_variable("postmirror_script") ); } print "\nPost Mirror script has completed. See above output for any possible errors.\n\n"; } unlock_aptmirror();