From 667a652a54587db56f3498371ed4e96a498c3e30 Mon Sep 17 00:00:00 2001 From: sneak Date: Wed, 22 May 2024 09:37:29 -0700 Subject: [PATCH] latest, time for testing --- LICENSE | 21 +++ fastmirror.go | 299 +++++++++++++++++++++++++-------------- go.mod | 8 ++ go.sum | 18 +++ sources_list_legacy.tmpl | 5 + sources_list_modern.tmpl | 14 ++ 6 files changed, 261 insertions(+), 104 deletions(-) create mode 100644 LICENSE create mode 100644 go.sum create mode 100644 sources_list_legacy.tmpl create mode 100644 sources_list_modern.tmpl diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..55c6f82 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Jeffrey Paul + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/fastmirror.go b/fastmirror.go index 0e021ff..f27284c 100644 --- a/fastmirror.go +++ b/fastmirror.go @@ -2,7 +2,10 @@ package fastmirror import ( "bufio" + _ "embed" "fmt" + "github.com/schollz/progressbar/v3" + "html/template" "io" "io/ioutil" "log" @@ -16,89 +19,116 @@ import ( "time" ) +//go:embed sources_list_legacy.tmpl +var legacyTemplate string + +//go:embed sources_list_modern.tmpl +var modernTemplate string + func CLIEntry() { + log.Println("Starting Fastmirror CLI") + if runtime.GOOS != "linux" { log.Fatal("This program is only for Linux") } - if !isUbuntu() { - log.Fatal("This program is only for Ubuntu") + if err := isUbuntu(); err != nil { + log.Fatal(err) } - sourcesFilePath := findSourcesFilePath() - - fmt.Printf("Found sources file: %s\n", sourcesFilePath) - - // Create backup of the sources file - createBackup(sourcesFilePath) - - // Create sources.list.d directory if it doesn't exist - createSourcesListD() - - // Fetch and find the fastest mirror - fastestMirrorURL := findFastestMirror() - fmt.Printf("Fastest mirror: %s\n", fastestMirrorURL) + sourcesFilePath, err := findSourcesFilePath() + if err != nil { + log.Fatal(err) + } + log.Printf("Found sources file: %s", sourcesFilePath) // Extract suites from the existing sources file - suites := extractSuites(sourcesFilePath) + suites, err := extractSuites(sourcesFilePath) + if err != nil { + log.Fatal(err) + } + log.Printf("Extracted suites: %v", suites) + + // Create backup of the sources file + if err := createBackup(sourcesFilePath); err != nil { + log.Fatal(err) + } + + // Create sources.list.d directory if it doesn't exist + if err := createSourcesListD(); err != nil { + log.Fatal(err) + } + + // Fetch and find the fastest mirror + fastestMirrorURL, fastestTime, latencies, err := findFastestMirror() + if err != nil { + log.Fatal(err) + } + log.Printf("Fastest mirror: %s", fastestMirrorURL) + + // Display latency statistics + displayLatencyStatistics(latencies, fastestMirrorURL, fastestTime) // Update sources file with the fastest mirror - updateSourcesFile(sourcesFilePath, fastestMirrorURL, suites) + if err := updateSourcesFile(sourcesFilePath, fastestMirrorURL, suites); err != nil { + log.Fatal(err) + } + log.Println("Fastmirror CLI finished successfully") } -func getUbuntuCodename() string { +func getUbuntuCodename() (string, error) { cmd := exec.Command("lsb_release", "-cs") output, err := cmd.Output() if err != nil { - log.Fatalf("Failed to run lsb_release: %v\n", err) + return "", fmt.Errorf("failed to run lsb_release: %v", err) } - codename := strings.TrimSpace(string(output)) - return codename + return strings.TrimSpace(string(output)), nil } -func getUbuntuVersion() (int, int) { +func getUbuntuVersion() (int, int, error) { cmd := exec.Command("lsb_release", "-rs") output, err := cmd.Output() if err != nil { - log.Fatalf("Failed to run lsb_release: %v\n", err) + return 0, 0, fmt.Errorf("failed to run lsb_release: %v", err) } version := strings.TrimSpace(string(output)) parts := strings.Split(version, ".") if len(parts) != 2 { - log.Fatalf("Unexpected version format: %s", version) + return 0, 0, fmt.Errorf("unexpected version format: %s", version) } majorVersion, err := strconv.Atoi(parts[0]) if err != nil { - log.Fatalf("Failed to parse major version: %v", err) + return 0, 0, fmt.Errorf("failed to parse major version: %v", err) } minorVersion, err := strconv.Atoi(parts[1]) if err != nil { - log.Fatalf("Failed to parse minor version: %v", err) + return 0, 0, fmt.Errorf("failed to parse minor version: %v", err) } - return majorVersion, minorVersion + return majorVersion, minorVersion, nil } -func isUbuntu() bool { +func isUbuntu() error { cmd := exec.Command("lsb_release", "-is") output, err := cmd.Output() if err != nil { - log.Fatalf("Failed to run lsb_release: %v\n", err) + return fmt.Errorf("failed to run lsb_release: %v", err) } - distro := strings.TrimSpace(string(output)) - return distro == "Ubuntu" + if strings.TrimSpace(string(output)) != "Ubuntu" { + return fmt.Errorf("this program is only for Ubuntu") + } + return nil } -func findSourcesFilePath() string { +func findSourcesFilePath() (string, error) { const sourcesListPath = "/etc/apt/sources.list" const sourcesListDPath = "/etc/apt/sources.list.d/" if isUbuntuSourcesFile(sourcesListPath) { - fmt.Printf("Found Ubuntu sources file: %s\n", sourcesListPath) - return sourcesListPath + log.Printf("Found Ubuntu sources file: %s", sourcesListPath) + return sourcesListPath, nil } files, err := ioutil.ReadDir(sourcesListDPath) if err != nil { - fmt.Printf("Failed to read directory %s: %v\n", sourcesListDPath, err) - os.Exit(1) + return "", fmt.Errorf("failed to read directory %s: %v", sourcesListDPath, err) } for _, file := range files { @@ -108,91 +138,114 @@ func findSourcesFilePath() string { filePath := filepath.Join(sourcesListDPath, file.Name()) if isUbuntuSourcesFile(filePath) { - fmt.Printf("Found Ubuntu sources file: %s\n", filePath) - return filePath + log.Printf("Found Ubuntu sources file: %s", filePath) + return filePath, nil } } - return "" + return "", fmt.Errorf("no Ubuntu sources file found") } func isUbuntuSourcesFile(filePath string) bool { content, err := ioutil.ReadFile(filePath) if err != nil { - fmt.Printf("Failed to read file %s: %v\n", filePath, err) + log.Printf("Failed to read file %s: %v", filePath, err) return false } return strings.Contains(strings.ToLower(string(content)), "ubuntu.com") } -func createBackup(filePath string) { +func createBackup(filePath string) error { backupPath := filePath + ".bak" - err := os.Rename(filePath, backupPath) - if err != nil { - log.Fatalf("Failed to create backup of %s: %v\n", filePath, err) + if _, err := os.Stat(backupPath); err == nil { + return fmt.Errorf("backup file %s already exists", backupPath) } - fmt.Printf("Created backup: %s\n", backupPath) + if err := os.Rename(filePath, backupPath); err != nil { + return fmt.Errorf("failed to create backup of %s: %v", filePath, err) + } + log.Printf("Created backup: %s", backupPath) + return nil } -func createSourcesListD() { +func createSourcesListD() error { const sourcesListDPath = "/etc/apt/sources.list.d/" if _, err := os.Stat(sourcesListDPath); os.IsNotExist(err) { - err = os.Mkdir(sourcesListDPath, 0755) - if err != nil { - log.Fatalf("Failed to create directory %s: %v\n", sourcesListDPath, err) + if err := os.Mkdir(sourcesListDPath, 0755); err != nil { + return fmt.Errorf("failed to create directory %s: %v", sourcesListDPath, err) } - fmt.Printf("Created directory: %s\n", sourcesListDPath) + log.Printf("Created directory: %s", sourcesListDPath) } + return nil } -func findFastestMirror() string { +func findFastestMirror() (string, time.Duration, []time.Duration, error) { + log.Println("Fetching mirrors list") resp, err := http.Get("http://mirrors.ubuntu.com/mirrors.txt") if err != nil { - log.Fatalf("Failed to fetch mirrors list: %v\n", err) + return "", 0, nil, fmt.Errorf("failed to fetch mirrors list: %v", err) } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { - log.Fatalf("Failed to read mirrors list: %v\n", err) + return "", 0, nil, fmt.Errorf("failed to read mirrors list: %v", err) } mirrors := strings.Split(string(body), "\n") + log.Printf("Found %d mirrors", len(mirrors)) + var fastestMirrorURL string var fastestTime time.Duration - codename := getUbuntuCodename() - architecture := runtime.GOARCH + var latencies []time.Duration + downCount := 0 + noIndexCount := 0 + + codename, err := getUbuntuCodename() + if err != nil { + return "", 0, nil, err + } httpClient := http.Client{ Timeout: 1 * time.Second, } + log.Println("Testing mirrors for latency") + bar := progressbar.Default(int64(len(mirrors))) + for _, mirror := range mirrors { + bar.Add(1) if strings.HasPrefix(mirror, "https://") { mirror = strings.TrimSuffix(mirror, "/") - if isValidMirror(httpClient, mirror, codename, architecture) { - startTime := time.Now() - resp, err := httpClient.Get(mirror) - if err == nil { - elapsedTime := time.Since(startTime) - resp.Body.Close() - if fastestMirrorURL == "" || elapsedTime < fastestTime { - fastestMirrorURL = mirror - fastestTime = elapsedTime - } + startTime := time.Now() + isValid := isValidMirror(httpClient, mirror, codename) + elapsedTime := time.Since(startTime) + + if isValid { + latencies = append(latencies, elapsedTime) + if fastestMirrorURL == "" || elapsedTime < fastestTime { + fastestMirrorURL = mirror + fastestTime = elapsedTime } + } else { + noIndexCount++ } + } else { + downCount++ } } if fastestMirrorURL == "" { - log.Fatal("No suitable HTTPS mirror found") + return "", 0, nil, fmt.Errorf("no suitable HTTPS mirror found") } - return fastestMirrorURL + displayLatencyStatistics(latencies, fastestMirrorURL, fastestTime) + log.Printf("Number of mirrors that were down: %d", downCount) + log.Printf("Number of mirrors without required package index: %d", noIndexCount) + + return fastestMirrorURL, fastestTime, latencies, nil } -func isValidMirror(httpClient http.Client, mirrorURL, codename, architecture string) bool { +func isValidMirror(httpClient http.Client, mirrorURL, codename string) bool { uri := fmt.Sprintf("%s/dists/%s/Release", mirrorURL, codename) resp, err := httpClient.Get(uri) if err != nil { @@ -202,14 +255,14 @@ func isValidMirror(httpClient http.Client, mirrorURL, codename, architecture str return resp.StatusCode == http.StatusOK } -func extractSuites(filePath string) []string { +func extractSuites(filePath string) ([]string, error) { content, err := ioutil.ReadFile(filePath) if err != nil { - log.Fatalf("Failed to read file %s: %v\n", filePath, err) + return nil, fmt.Errorf("failed to read file %s: %v", filePath, err) } // Extract suites from the sources file - suites := []string{} + var suites []string scanner := bufio.NewScanner(strings.NewReader(string(content))) for scanner.Scan() { line := scanner.Text() @@ -221,52 +274,90 @@ func extractSuites(filePath string) []string { } } if err := scanner.Err(); err != nil { - log.Fatalf("Failed to scan file %s: %v\n", filePath, err) + return nil, fmt.Errorf("failed to scan file %s: %v", filePath, err) } - return suites + return suites, nil } -func updateSourcesFile(filePath, mirrorURL string, suites []string) { - architecture := runtime.GOARCH - codename := getUbuntuCodename() - majorVersion, minorVersion := getUbuntuVersion() +func displayLatencyStatistics(latencies []time.Duration, fastestMirrorURL string, fastestTime time.Duration) { + if len(latencies) == 0 { + log.Println("No latencies recorded") + return + } - var uriPath string - if architecture == "amd64" || architecture == "386" { - uriPath = "/ubuntu" - } else { - uriPath = "/ubuntu-ports" + var totalLatency time.Duration + var minLatency time.Duration = latencies[0] + var maxLatency time.Duration = latencies[0] + + for _, latency := range latencies { + totalLatency += latency + if latency < minLatency { + minLatency = latency + } + if latency > maxLatency { + maxLatency = latency + } + } + + averageLatency := totalLatency / time.Duration(len(latencies)) + + log.Printf("Minimum latency: %s", minLatency) + log.Printf("Maximum latency: %s", maxLatency) + log.Printf("Average latency: %s", averageLatency) + log.Printf("Chosen fastest mirror: %s with latency: %s", fastestMirrorURL, fastestTime) +} + +func updateSourcesFile(filePath, mirrorURL string, suites []string) error { + codename, err := getUbuntuCodename() + if err != nil { + return err + } + majorVersion, minorVersion, err := getUbuntuVersion() + if err != nil { + return err } suitesString := strings.Join(suites, " ") var content string + data := struct { + MirrorURL string + Codename string + Suites string + }{ + MirrorURL: mirrorURL, + Codename: codename, + Suites: suitesString, + } + if majorVersion > 24 || (majorVersion == 24 && minorVersion >= 4) { - content = fmt.Sprintf(`Types: deb -URIs: %s%s -Suites: %s -Components: %s -Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg - -## Ubuntu security updates. Aside from URIs and Suites, -## this should mirror your choices in the previous section. -Types: deb -URIs: %s%s -Suites: %s-security -Components: %s -Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg -`, mirrorURL, uriPath, codename, suitesString, mirrorURL, uriPath, codename, suitesString) + tmpl, err := template.New("modern").Parse(modernTemplate) + if err != nil { + return fmt.Errorf("failed to parse modern template: %v", err) + } + var buf strings.Builder + err = tmpl.Execute(&buf, data) + if err != nil { + return fmt.Errorf("failed to execute modern template: %v", err) + } + content = buf.String() } else { - content = fmt.Sprintf(`deb %s%s %s %s -deb %s%s %s-updates %s -deb %s%s %s-backports %s -deb %s%s %s-security %s -`, mirrorURL, uriPath, codename, suitesString, mirrorURL, uriPath, codename, suitesString, mirrorURL, uriPath, codename, suitesString, mirrorURL, uriPath, codename, suitesString) + tmpl, err := template.New("legacy").Parse(legacyTemplate) + if err != nil { + return fmt.Errorf("failed to parse legacy template: %v", err) + } + var buf strings.Builder + err = tmpl.Execute(&buf, data) + if err != nil { + return fmt.Errorf("failed to execute legacy template: %v", err) + } + content = buf.String() } - err := ioutil.WriteFile(filePath, []byte(content), 0644) + err = ioutil.WriteFile(filePath, []byte(content), 0644) if err != nil { - log.Fatalf("Failed to update sources file %s: %v\n", filePath, err) + return fmt.Errorf("failed to update sources file %s: %v", filePath, err) } - fmt.Printf("Updated sources file: %s\n", filePath) + log.Printf("Updated sources file: %s", filePath) + return nil } diff --git a/go.mod b/go.mod index 191a6b2..6d17566 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,11 @@ module git.eeqj.de/sneak/fastmirror go 1.22.2 + +require ( + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/schollz/progressbar/v3 v3.14.3 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/term v0.20.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..af4bd7b --- /dev/null +++ b/go.sum @@ -0,0 +1,18 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/schollz/progressbar/v3 v3.14.3 h1:oOuWW19ka12wxYU1XblR4n16wF/2Y1dBLMarMo6p4xU= +github.com/schollz/progressbar/v3 v3.14.3/go.mod h1:aT3UQ7yGm+2ZjeXPqsjTenwL3ddUiuZ0kfQ/2tHlyNI= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= diff --git a/sources_list_legacy.tmpl b/sources_list_legacy.tmpl new file mode 100644 index 0000000..b674018 --- /dev/null +++ b/sources_list_legacy.tmpl @@ -0,0 +1,5 @@ +deb {{.MirrorURL}}{{.URIPath}} {{.Codename}} {{.Suites}} +deb {{.MirrorURL}}{{.URIPath}} {{.Codename}}-updates {{.Suites}} +deb {{.MirrorURL}}{{.URIPath}} {{.Codename}}-backports {{.Suites}} +deb {{.MirrorURL}}{{.URIPath}} {{.Codename}}-security {{.Suites}} + diff --git a/sources_list_modern.tmpl b/sources_list_modern.tmpl new file mode 100644 index 0000000..66d09ea --- /dev/null +++ b/sources_list_modern.tmpl @@ -0,0 +1,14 @@ +Types: deb +URIs: {{.MirrorURL}}{{.URIPath}} +Suites: {{.Codename}} {{.Codename}}-updates {{.Codename}}-backports +Components: {{.Suites}} +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + +## Ubuntu security updates. Aside from URIs and Suites, +## this should mirror your choices in the previous section. +Types: deb +URIs: {{.MirrorURL}}{{.URIPath}} +Suites: {{.Codename}}-security +Components: {{.Suites}} +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +