sneak
/
fastmirror
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

latest, time for testing

This commit is contained in:
Jeffrey Paul 2024-05-22 09:37:29 -07:00
parent a0980a4c67
commit 667a652a54
6 changed files with 261 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
LICENSE Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2024 Jeffrey Paul
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

297
fastmirror.go
View File

@ -2,7 +2,10 @@ package fastmirror
import ( import (
"bufio" "bufio"
_ "embed"
"fmt" "fmt"
"github.com/schollz/progressbar/v3"
"html/template"
"io" "io"
"io/ioutil" "io/ioutil"
"log" "log"
@ -16,89 +19,116 @@ import (
"time" "time"
) )
//go:embed sources_list_legacy.tmpl
var legacyTemplate string
//go:embed sources_list_modern.tmpl
var modernTemplate string
func CLIEntry() { func CLIEntry() {
log.Println("Starting Fastmirror CLI")
if runtime.GOOS != "linux" { if runtime.GOOS != "linux" {
log.Fatal("This program is only for Linux") log.Fatal("This program is only for Linux")
} }
if !isUbuntu() { if err := isUbuntu(); err != nil {
log.Fatal("This program is only for Ubuntu") log.Fatal(err)
} }
sourcesFilePath := findSourcesFilePath() sourcesFilePath, err := findSourcesFilePath()
if err != nil {
fmt.Printf("Found sources file: %s\n", sourcesFilePath) log.Fatal(err)
}
// Create backup of the sources file log.Printf("Found sources file: %s", sourcesFilePath)
createBackup(sourcesFilePath)
// Create sources.list.d directory if it doesn't exist
createSourcesListD()
// Fetch and find the fastest mirror
fastestMirrorURL := findFastestMirror()
fmt.Printf("Fastest mirror: %s\n", fastestMirrorURL)
// Extract suites from the existing sources file // Extract suites from the existing sources file
suites := extractSuites(sourcesFilePath) suites, err := extractSuites(sourcesFilePath)
if err != nil {
log.Fatal(err)
}
log.Printf("Extracted suites: %v", suites)
// Update sources file with the fastest mirror // Create backup of the sources file
updateSourcesFile(sourcesFilePath, fastestMirrorURL, suites) if err := createBackup(sourcesFilePath); err != nil {
log.Fatal(err)
} }
func getUbuntuCodename() string { // Create sources.list.d directory if it doesn't exist
if err := createSourcesListD(); err != nil {
log.Fatal(err)
}
// Fetch and find the fastest mirror
fastestMirrorURL, fastestTime, latencies, err := findFastestMirror()
if err != nil {
log.Fatal(err)
}
log.Printf("Fastest mirror: %s", fastestMirrorURL)
// Display latency statistics
displayLatencyStatistics(latencies, fastestMirrorURL, fastestTime)
// Update sources file with the fastest mirror
if err := updateSourcesFile(sourcesFilePath, fastestMirrorURL, suites); err != nil {
log.Fatal(err)
}
log.Println("Fastmirror CLI finished successfully")
}
func getUbuntuCodename() (string, error) {
cmd := exec.Command("lsb_release", "-cs") cmd := exec.Command("lsb_release", "-cs")
output, err := cmd.Output() output, err := cmd.Output()
if err != nil { if err != nil {
log.Fatalf("Failed to run lsb_release: %v\n", err) return "", fmt.Errorf("failed to run lsb_release: %v", err)
} }
codename := strings.TrimSpace(string(output)) return strings.TrimSpace(string(output)), nil
return codename
} }
func getUbuntuVersion() (int, int) { func getUbuntuVersion() (int, int, error) {
cmd := exec.Command("lsb_release", "-rs") cmd := exec.Command("lsb_release", "-rs")
output, err := cmd.Output() output, err := cmd.Output()
if err != nil { if err != nil {
log.Fatalf("Failed to run lsb_release: %v\n", err) return 0, 0, fmt.Errorf("failed to run lsb_release: %v", err)
} }
version := strings.TrimSpace(string(output)) version := strings.TrimSpace(string(output))
parts := strings.Split(version, ".") parts := strings.Split(version, ".")
if len(parts) != 2 { if len(parts) != 2 {
log.Fatalf("Unexpected version format: %s", version) return 0, 0, fmt.Errorf("unexpected version format: %s", version)
} }
majorVersion, err := strconv.Atoi(parts[0]) majorVersion, err := strconv.Atoi(parts[0])
if err != nil { if err != nil {
log.Fatalf("Failed to parse major version: %v", err) return 0, 0, fmt.Errorf("failed to parse major version: %v", err)
} }
minorVersion, err := strconv.Atoi(parts[1]) minorVersion, err := strconv.Atoi(parts[1])
if err != nil { if err != nil {
log.Fatalf("Failed to parse minor version: %v", err) return 0, 0, fmt.Errorf("failed to parse minor version: %v", err)
} }
return majorVersion, minorVersion return majorVersion, minorVersion, nil
} }
func isUbuntu() bool { func isUbuntu() error {
cmd := exec.Command("lsb_release", "-is") cmd := exec.Command("lsb_release", "-is")
output, err := cmd.Output() output, err := cmd.Output()
if err != nil { if err != nil {
log.Fatalf("Failed to run lsb_release: %v\n", err) return fmt.Errorf("failed to run lsb_release: %v", err)
} }
distro := strings.TrimSpace(string(output)) if strings.TrimSpace(string(output)) != "Ubuntu" {
return distro == "Ubuntu" return fmt.Errorf("this program is only for Ubuntu")
}
return nil
} }
func findSourcesFilePath() string { func findSourcesFilePath() (string, error) {
const sourcesListPath = "/etc/apt/sources.list" const sourcesListPath = "/etc/apt/sources.list"
const sourcesListDPath = "/etc/apt/sources.list.d/" const sourcesListDPath = "/etc/apt/sources.list.d/"
if isUbuntuSourcesFile(sourcesListPath) { if isUbuntuSourcesFile(sourcesListPath) {
fmt.Printf("Found Ubuntu sources file: %s\n", sourcesListPath) log.Printf("Found Ubuntu sources file: %s", sourcesListPath)
return sourcesListPath return sourcesListPath, nil
} }
files, err := ioutil.ReadDir(sourcesListDPath) files, err := ioutil.ReadDir(sourcesListDPath)
if err != nil { if err != nil {
fmt.Printf("Failed to read directory %s: %v\n", sourcesListDPath, err) return "", fmt.Errorf("failed to read directory %s: %v", sourcesListDPath, err)
os.Exit(1)
} }
for _, file := range files { for _, file := range files {
@ -108,91 +138,114 @@ func findSourcesFilePath() string {
filePath := filepath.Join(sourcesListDPath, file.Name()) filePath := filepath.Join(sourcesListDPath, file.Name())
if isUbuntuSourcesFile(filePath) { if isUbuntuSourcesFile(filePath) {
fmt.Printf("Found Ubuntu sources file: %s\n", filePath) log.Printf("Found Ubuntu sources file: %s", filePath)
return filePath return filePath, nil
} }
} }
return "" return "", fmt.Errorf("no Ubuntu sources file found")
} }
func isUbuntuSourcesFile(filePath string) bool { func isUbuntuSourcesFile(filePath string) bool {
content, err := ioutil.ReadFile(filePath) content, err := ioutil.ReadFile(filePath)
if err != nil { if err != nil {
fmt.Printf("Failed to read file %s: %v\n", filePath, err) log.Printf("Failed to read file %s: %v", filePath, err)
return false return false
} }
return strings.Contains(strings.ToLower(string(content)), "ubuntu.com") return strings.Contains(strings.ToLower(string(content)), "ubuntu.com")
} }
func createBackup(filePath string) { func createBackup(filePath string) error {
backupPath := filePath + ".bak" backupPath := filePath + ".bak"
err := os.Rename(filePath, backupPath) if _, err := os.Stat(backupPath); err == nil {
if err != nil { return fmt.Errorf("backup file %s already exists", backupPath)
log.Fatalf("Failed to create backup of %s: %v\n", filePath, err)
} }
fmt.Printf("Created backup: %s\n", backupPath) if err := os.Rename(filePath, backupPath); err != nil {
return fmt.Errorf("failed to create backup of %s: %v", filePath, err)
}
log.Printf("Created backup: %s", backupPath)
return nil
} }
func createSourcesListD() { func createSourcesListD() error {
const sourcesListDPath = "/etc/apt/sources.list.d/" const sourcesListDPath = "/etc/apt/sources.list.d/"
if _, err := os.Stat(sourcesListDPath); os.IsNotExist(err) { if _, err := os.Stat(sourcesListDPath); os.IsNotExist(err) {
err = os.Mkdir(sourcesListDPath, 0755) if err := os.Mkdir(sourcesListDPath, 0755); err != nil {
if err != nil { return fmt.Errorf("failed to create directory %s: %v", sourcesListDPath, err)
log.Fatalf("Failed to create directory %s: %v\n", sourcesListDPath, err)
} }
fmt.Printf("Created directory: %s\n", sourcesListDPath) log.Printf("Created directory: %s", sourcesListDPath)
} }
return nil
} }
func findFastestMirror() string { func findFastestMirror() (string, time.Duration, []time.Duration, error) {
log.Println("Fetching mirrors list")
resp, err := http.Get("http://mirrors.ubuntu.com/mirrors.txt") resp, err := http.Get("http://mirrors.ubuntu.com/mirrors.txt")
if err != nil { if err != nil {
log.Fatalf("Failed to fetch mirrors list: %v\n", err) return "", 0, nil, fmt.Errorf("failed to fetch mirrors list: %v", err)
} }
defer resp.Body.Close() defer resp.Body.Close()
body, err := io.ReadAll(resp.Body) body, err := io.ReadAll(resp.Body)
if err != nil { if err != nil {
log.Fatalf("Failed to read mirrors list: %v\n", err) return "", 0, nil, fmt.Errorf("failed to read mirrors list: %v", err)
} }
mirrors := strings.Split(string(body), "\n") mirrors := strings.Split(string(body), "\n")
log.Printf("Found %d mirrors", len(mirrors))
var fastestMirrorURL string var fastestMirrorURL string
var fastestTime time.Duration var fastestTime time.Duration
codename := getUbuntuCodename() var latencies []time.Duration
architecture := runtime.GOARCH downCount := 0
noIndexCount := 0
codename, err := getUbuntuCodename()
if err != nil {
return "", 0, nil, err
}
httpClient := http.Client{ httpClient := http.Client{
Timeout: 1 * time.Second, Timeout: 1 * time.Second,
} }
log.Println("Testing mirrors for latency")
bar := progressbar.Default(int64(len(mirrors)))
for _, mirror := range mirrors { for _, mirror := range mirrors {
bar.Add(1)
if strings.HasPrefix(mirror, "https://") { if strings.HasPrefix(mirror, "https://") {
mirror = strings.TrimSuffix(mirror, "/") mirror = strings.TrimSuffix(mirror, "/")
if isValidMirror(httpClient, mirror, codename, architecture) {
startTime := time.Now() startTime := time.Now()
resp, err := httpClient.Get(mirror) isValid := isValidMirror(httpClient, mirror, codename)
if err == nil {
elapsedTime := time.Since(startTime) elapsedTime := time.Since(startTime)
resp.Body.Close()
if isValid {
latencies = append(latencies, elapsedTime)
if fastestMirrorURL == "" || elapsedTime < fastestTime { if fastestMirrorURL == "" || elapsedTime < fastestTime {
fastestMirrorURL = mirror fastestMirrorURL = mirror
fastestTime = elapsedTime fastestTime = elapsedTime
} }
} else {
noIndexCount++
} }
} } else {
downCount++
} }
} }
if fastestMirrorURL == "" { if fastestMirrorURL == "" {
log.Fatal("No suitable HTTPS mirror found") return "", 0, nil, fmt.Errorf("no suitable HTTPS mirror found")
} }
return fastestMirrorURL displayLatencyStatistics(latencies, fastestMirrorURL, fastestTime)
log.Printf("Number of mirrors that were down: %d", downCount)
log.Printf("Number of mirrors without required package index: %d", noIndexCount)
return fastestMirrorURL, fastestTime, latencies, nil
} }
func isValidMirror(httpClient http.Client, mirrorURL, codename, architecture string) bool { func isValidMirror(httpClient http.Client, mirrorURL, codename string) bool {
uri := fmt.Sprintf("%s/dists/%s/Release", mirrorURL, codename) uri := fmt.Sprintf("%s/dists/%s/Release", mirrorURL, codename)
resp, err := httpClient.Get(uri) resp, err := httpClient.Get(uri)
if err != nil { if err != nil {
@ -202,14 +255,14 @@ func isValidMirror(httpClient http.Client, mirrorURL, codename, architecture str
return resp.StatusCode == http.StatusOK return resp.StatusCode == http.StatusOK
} }
func extractSuites(filePath string) []string { func extractSuites(filePath string) ([]string, error) {
content, err := ioutil.ReadFile(filePath) content, err := ioutil.ReadFile(filePath)
if err != nil { if err != nil {
log.Fatalf("Failed to read file %s: %v\n", filePath, err) return nil, fmt.Errorf("failed to read file %s: %v", filePath, err)
} }
// Extract suites from the sources file // Extract suites from the sources file
suites := []string{} var suites []string
scanner := bufio.NewScanner(strings.NewReader(string(content))) scanner := bufio.NewScanner(strings.NewReader(string(content)))
for scanner.Scan() { for scanner.Scan() {
line := scanner.Text() line := scanner.Text()
@ -221,52 +274,90 @@ func extractSuites(filePath string) []string {
} }
} }
if err := scanner.Err(); err != nil { if err := scanner.Err(); err != nil {
log.Fatalf("Failed to scan file %s: %v\n", filePath, err) return nil, fmt.Errorf("failed to scan file %s: %v", filePath, err)
} }
return suites return suites, nil
} }
func updateSourcesFile(filePath, mirrorURL string, suites []string) { func displayLatencyStatistics(latencies []time.Duration, fastestMirrorURL string, fastestTime time.Duration) {
architecture := runtime.GOARCH if len(latencies) == 0 {
codename := getUbuntuCodename() log.Println("No latencies recorded")
majorVersion, minorVersion := getUbuntuVersion() return
}
var uriPath string var totalLatency time.Duration
if architecture == "amd64" || architecture == "386" { var minLatency time.Duration = latencies[0]
uriPath = "/ubuntu" var maxLatency time.Duration = latencies[0]
} else {
uriPath = "/ubuntu-ports" for _, latency := range latencies {
totalLatency += latency
if latency < minLatency {
minLatency = latency
}
if latency > maxLatency {
maxLatency = latency
}
}
averageLatency := totalLatency / time.Duration(len(latencies))
log.Printf("Minimum latency: %s", minLatency)
log.Printf("Maximum latency: %s", maxLatency)
log.Printf("Average latency: %s", averageLatency)
log.Printf("Chosen fastest mirror: %s with latency: %s", fastestMirrorURL, fastestTime)
}
func updateSourcesFile(filePath, mirrorURL string, suites []string) error {
codename, err := getUbuntuCodename()
if err != nil {
return err
}
majorVersion, minorVersion, err := getUbuntuVersion()
if err != nil {
return err
} }
suitesString := strings.Join(suites, " ") suitesString := strings.Join(suites, " ")
var content string var content string
data := struct {
MirrorURL string
Codename string
Suites string
}{
MirrorURL: mirrorURL,
Codename: codename,
Suites: suitesString,
}
if majorVersion > 24 || (majorVersion == 24 && minorVersion >= 4) { if majorVersion > 24 || (majorVersion == 24 && minorVersion >= 4) {
content = fmt.Sprintf(`Types: deb tmpl, err := template.New("modern").Parse(modernTemplate)
URIs: %s%s
Suites: %s
Components: %s
Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
## Ubuntu security updates. Aside from URIs and Suites,
## this should mirror your choices in the previous section.
Types: deb
URIs: %s%s
Suites: %s-security
Components: %s
Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
`, mirrorURL, uriPath, codename, suitesString, mirrorURL, uriPath, codename, suitesString)
} else {
content = fmt.Sprintf(`deb %s%s %s %s
deb %s%s %s-updates %s
deb %s%s %s-backports %s
deb %s%s %s-security %s
`, mirrorURL, uriPath, codename, suitesString, mirrorURL, uriPath, codename, suitesString, mirrorURL, uriPath, codename, suitesString, mirrorURL, uriPath, codename, suitesString)
}
err := ioutil.WriteFile(filePath, []byte(content), 0644)
if err != nil { if err != nil {
log.Fatalf("Failed to update sources file %s: %v\n", filePath, err) return fmt.Errorf("failed to parse modern template: %v", err)
} }
fmt.Printf("Updated sources file: %s\n", filePath) var buf strings.Builder
err = tmpl.Execute(&buf, data)
if err != nil {
return fmt.Errorf("failed to execute modern template: %v", err)
}
content = buf.String()
} else {
tmpl, err := template.New("legacy").Parse(legacyTemplate)
if err != nil {
return fmt.Errorf("failed to parse legacy template: %v", err)
}
var buf strings.Builder
err = tmpl.Execute(&buf, data)
if err != nil {
return fmt.Errorf("failed to execute legacy template: %v", err)
}
content = buf.String()
}
err = ioutil.WriteFile(filePath, []byte(content), 0644)
if err != nil {
return fmt.Errorf("failed to update sources file %s: %v", filePath, err)
}
log.Printf("Updated sources file: %s", filePath)
return nil
} }

8
go.mod
View File

@ -1,3 +1,11 @@
module git.eeqj.de/sneak/fastmirror module git.eeqj.de/sneak/fastmirror
go 1.22.2 go 1.22.2
require (
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/schollz/progressbar/v3 v3.14.3 // indirect
golang.org/x/sys v0.20.0 // indirect
golang.org/x/term v0.20.0 // indirect
)

18
go.sum Normal file
View File

@ -0,0 +1,18 @@
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/schollz/progressbar/v3 v3.14.3 h1:oOuWW19ka12wxYU1XblR4n16wF/2Y1dBLMarMo6p4xU=
github.com/schollz/progressbar/v3 v3.14.3/go.mod h1:aT3UQ7yGm+2ZjeXPqsjTenwL3ddUiuZ0kfQ/2tHlyNI=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=

5
sources_list_legacy.tmpl Normal file
View File

@ -0,0 +1,5 @@
deb {{.MirrorURL}}{{.URIPath}} {{.Codename}} {{.Suites}}
deb {{.MirrorURL}}{{.URIPath}} {{.Codename}}-updates {{.Suites}}
deb {{.MirrorURL}}{{.URIPath}} {{.Codename}}-backports {{.Suites}}
deb {{.MirrorURL}}{{.URIPath}} {{.Codename}}-security {{.Suites}}

14
sources_list_modern.tmpl Normal file
View File

@ -0,0 +1,14 @@
Types: deb
URIs: {{.MirrorURL}}{{.URIPath}}
Suites: {{.Codename}} {{.Codename}}-updates {{.Codename}}-backports
Components: {{.Suites}}
Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
## Ubuntu security updates. Aside from URIs and Suites,
## this should mirror your choices in the previous section.
Types: deb
URIs: {{.MirrorURL}}{{.URIPath}}
Suites: {{.Codename}}-security
Components: {{.Suites}}
Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg