Ayant du mal à comprendre le fonctionnement et surtout comment gérer les options de pacman-mirrors (il faut un doctorat en informatique !), je me suis donc amusé à écrire une alternative beaucoup plus simple
- Il est basé sur la prochaine version de pacman-mirrors avec des appels asynchrones - Ce qui le rend extrêmement rapide.
Si rapide qu'il n'y a plus lieu de sélectionner un pays : donc on ne sélectionne (automatiquement) qu'un continent - Il utilise geoIP pour trouver le pays et continent du demandeur, mais il est possible de sélectionner un continent manuellement
- Il sélectionne uniquement les miroirs à jour de notre continent
- Il sélectionne les miroirs de notre branche (pas testé car je n'ai que stable !!!)
- Il met en tête de liste les miroirs de notre pays (à cause du timeur...)
- Il lance les tests en parallèle et se stoppe au bout de 5 secondes (par défaut)
- Il sauvegarde uniquement si il est lancé avec "sudo", il fait une sauvegarde avant
90 mirrors (all)
52 mirrors in continent: Europe
24 mirrors in branch: stable
sort by continent (first:Europe) and country (first:France)
...
Resume all
0.18645739555358887 http://kibo.remi.lu/
0.26831841468811035 http://mirror.catn.com/pub/manjaro/
0.32701778411865234 http://manjaro.mirrors.uk2.net/
0.34636378288269043 http://ftp.free.org/mirrors/repo.manjaro.org/repos/
0.39118218421936035 http://ftp.snt.utwente.nl/pub/linux/manjaro/
0.3985757827758789 http://mirror.easyname.at/manjaro/
0.4525022506713867 http://mirror.datacenter.by/pub/mirrors/manjaro/
0.46449899673461914 http://mirrors.colocall.net/manjaro/
0.4885101318359375 http://ftp.vectranet.pl/manjaro/
0.5062437057495117 http://mirror.futureweb.be/manjaro/
0.5090365409851074 http://mirror.terrahost.no/linux/manjaro/
0.5160796642303467 http://ftp.tu-chemnitz.de/pub/linux/manjaro/
0.6179747581481934 https://mirror.neostrada.nl/manjaro/
0.6668210029602051 https://manjaro.moson.eu/
0.837090015411377 http://mirror.truenetwork.ru/manjaro/
0.9191088676452637 http://mirror.inode.at/manjaro/
1.0456562042236328 https://quantum-mirror.hu/mirrors/pub/manjaro/
1.12028169631958 https://nova.quantum-mirror.hu/mirrors/pub/manjaro/
1.1471326351165771 https://ftp.caliu.cat/pub/distribucions/manjaro/
1.2209060192108154 https://mirror.23media.com/manjaro/
1.6292178630828857 http://mirror.ragenetwork.de/manjaro/
2.285478353500366 https://manjaro.grena.ge/
-- http://ftp.linux.org.tr/manjaro/
-- http://repo.manjaro.org.uk/
::Total duration: 4.007263660430908
Saved in /etc/pacman.d/mirrorlist, backup in /etc/pacman.d/mirrorlist~
Pour "jouer", il est possible de passer quelques paramètre :
./mirrors-update.py AS
Si le timeout de 5 secondes est trop petit , il est possible de lui en passer un différent :
./mirrors-update 16
./mirrors-update --log
./mirrors-update -a
Il ne teste que l'url et pas la validité du protocole (https,http,ftp)
Il ne modifie pas la branche
---
#!/usr/bin/python
"""
mirrors-update.py
v 1.0.1 - 27/10/2019
"""
import sys
import os
from urllib import request
import json
import time
import datetime
import locale
from enum import IntEnum
MAXDOWNLOAD = 50000 # read only 50 000 octets
max_workers = 8
continents = {
"AF": "Africa",
"AN": "Antarctica",
"AS": "Asia",
"EU": "Europe",
"NA": "North America",
"OC": "Oceania",
"SA": "South America"
}
countries = {
'Australia':'OC',
'Bangladesh':'AS',
'Brazil':'SA',
'Canada':'NA',
'Chile':'SA',
'China':'AS',
'Costa_Rica':'SA',
'Colombia':'SA',
'Ecuador':'SA',
'Hong_Kong':'AS',
'India':'AS',
'Indonesia':'AS',
'Iran':'AS',
'Japan':'AS',
'Kenya':'AF',
'New_Zealand':'OC',
'Philippines':'AS',
'Singapore':'AS',
'South_Africa':'AF',
'South_Korea':'AS',
'Taiwan':'AS',
'Thailand':'AS',
'United_States':'NA',
'Vietnam':'AS',
}
class Branches(IntEnum):
stable = 0
testing = 1
unstable = 2
config = {
'country': 'France',
'continent_code': 'EU',
'continent': 'Europe',
'max_workers': 8,
'debug' : False,
'log': False,
'timeout' : 5, # stop all tests after X seconds
'branch': Branches.stable,
'count': 3, # want only 3 first speed
}
def usage():
print("Usage: ./mirrors-update.py [continent code] [-a] [timeout]")
print("-a: \t\t all mirrors")
print("continent code: \t filter (EU,AS,AF,OC,NA,SA)")
print("timeout: for stop tests (default 4 seconds)")
print("")
print("ENV: \"count\": save x mirrors (3 by default)")
exit(0)
def get_parameters(config):
if '-h' in sys.argv or '--help' in sys.argv:
usage()
if '--debug' in sys.argv:
config['debug'] = True
config['log'] = True
if '--log' in sys.argv:
config['log'] = True
config['count'] = int(os.getenv('count', 3))
config['country'], config['continent_code'] = get_my_position()
for arg in sys.argv:
if arg in continents.keys():
config['continent_code'] = arg
config['continent'] = continents.get(arg)
for arg in sys.argv:
if arg.isdigit():
config['timeout'] = int(arg)
with open("/etc/pacman-mirrors.conf") as conf_file:
for line in conf_file:
if line.startswith("Branch"):
(_, value) = line.split("=", 1)
value = value.strip()
try:
config['branch'] = Branches(Branches._member_names_.index(value))
except ValueError:
pass
return config
def get_my_position(ip='') ->str:
""" param ip : only for tests """
if ip:
ip = f"/{ip}"
try:
with request.urlopen(f"https://get.geojs.io/v1/ip/geo{ip}.json") as f_url:
req = json.loads(f_url.read())
return req['country'], req['continent_code']
except Exception:
return '', ''
return '', ''
def filter_continent(mirrors: list, continent: str) ->list:
if not continent:
return mirrors
# or can use ? pacman_mirrors/constants/timezones.py
return [m for m in mirrors if countries.get(m['country'], 'EU') == continent]
def load_datas(url: str) -> list:
""" load url list """
with request.urlopen(url) as f_url:
req = f_url.read()
mirrors = json.loads(req)
print(f"{len(mirrors)} mirrors (all)")
for mirror in mirrors:
mirror['timer'] = 99 # by default all bad
mirror['continent_code'] = countries.get(mirror['country'], 'EU')
return mirrors
def download_data(mirror: dict)->int:
""" NOT use protocols !"""
url = f"{mirror['url']}stable/core/x86_64/core.db"
start = time.time()
with request.urlopen(url, timeout=3) as f_url:
_ = f_url.read(MAXDOWNLOAD)
end = time.time()-start
mirror['timer'] = end
print(f" => {end:.3f}\t{mirror['url']}")
return end
def test_threads(timeout: int) ->int:
from concurrent.futures import ThreadPoolExecutor
import concurrent.futures
print("::Speed tests...")
startthread = time.time()
with ThreadPoolExecutor(max_workers=max_workers) as executor:
future_to_url = {executor.submit(download_data, url): url for url in mirrors}
try:
for _ in concurrent.futures.as_completed(future_to_url, timeout=timeout):
pass
except KeyboardInterrupt:
executor._threads.clear()
concurrent.futures.thread._threads_queues.clear()
return time.time()-startthread
except concurrent.futures._base.TimeoutError as err:
print(f"::Timeout: {err}", file=sys.stderr)
executor._threads.clear()
concurrent.futures.thread._threads_queues.clear()
end = time.time()-startthread
print(f"::Total duration: {end}")
return end
def save_log(mirrors: list, url: str, title: str):
""" save in log for view if we have always the same first url ..."""
mirrors.sort(key=lambda mirror: mirror['timer'], reverse=False)
with open(url, 'a') as flog:
flog.write(f"{title}\t")
for i, mirror in enumerate(mirrors):
flog.write(f"{mirror['url']}\t")
if i > 2:
break
flog.write("\n")
def save_mirrorlist(filename: str, mirrors: list, branch: str, maxi=12)->bool:
""" save bests in etc """
from shutil import copyfile
if not mirrors:
return False
if maxi < 1:
maxi = 1
try:
copyfile(filename, f"{filename}~")
except Exception as err:
print("ERROR:", err, file=sys.stderr)
sys.exit(2)
try:
with open(filename, "w") as outfile:
outfile.write(f"##\n# Manjaro Linux default mirrorlist\n# {datetime.date.today().strftime('%x')}\n##\n\n")
for i, mirror in enumerate(mirrors):
if i >= maxi:
continue
country = mirror['country']
if country.startswith('!'):
country = country[1:]
outfile.write(f"## Country : {country}\n")
outfile.write(f"Server = {mirror['url']}{branch}/$repo/$arch\n\n")
except OSError as err:
print("ERROR:", err, file=sys.stderr)
sys.exit(2)
print(f"\n::Saved in {filename}, backup in {filename}~")
return True
if __name__ == '__main__':
locale.setlocale(locale.LC_ALL, '')
config = get_parameters(config)
print("parameters:", config)
mirrors = load_datas("https://repo.manjaro.org/status.json")
#print(mirrors)
if '-a' not in sys.argv:
mirrors = filter_continent(mirrors, config['continent_code'])
print(f"{len(mirrors)} mirrors in continent: {config['continent']}")
mirrors = [m for m in mirrors if m['branches'][int(config['branch'])] == 1 and 'http' in m['protocols']]
print(f"{len(mirrors)} mirrors in branch: {config['branch'].name}")
print(f"sort by continent (first:{config['continent']}) and country (first:{config['country']})")
for mirror in mirrors:
if mirror['country'] == config['country']:
mirror['country'] = f"!{mirror['country']}"
if mirror['continent_code'] == config['continent_code']:
mirror['continent_code'] = f"!{mirror['continent_code']}"
mirrors.sort(key=lambda mirror: mirror['country'], reverse=False)
mirrors.sort(key=lambda mirror: mirror['continent_code'], reverse=False)
#print(mirrors)
duration = test_threads(config['timeout'])
# sort mirrors by timers """
mirrors.sort(key=lambda mirror: mirror['timer'], reverse=False)
print("\n::Resume all")
for mirror in mirrors:
if mirror['timer'] > 90:
print(f"-- \t{mirror['url']}", file=sys.stderr)
else:
print(f"{mirror['timer']:.3f}\t{mirror['url']}")
print(f"::Total duration: {duration}")
mirrors = [m for m in mirrors if int(m['timer']) < 30]
if config['log']:
save_log(mirrors, "./logs.txt", duration)
if os.geteuid() == 0 and mirrors:
# if root save in /etc/pacman.d/mirrorlist
save_mirrorlist("/etc/pacman.d/mirrorlist", mirrors, config['branch'].name, config['count'])