diff --git a/cmd/dget/main.go b/cmd/dget/main.go new file mode 100644 index 0000000..1337dc1 --- /dev/null +++ b/cmd/dget/main.go @@ -0,0 +1,45 @@ +package main + +import ( + "flag" + "os" + "strings" + + "gitee.com/extrame/dget" + "github.com/sirupsen/logrus" +) + +func main() { + debug := flag.Bool("debug", false, "打印调试信息") + printInfo := flag.Bool("print", false, "只打印获取信息") + arch := flag.String("arch", "linux/amd64", "指定架构") + + flag.Parse() + + if *debug { + dget.SetLogLevel(logrus.DebugLevel) + } + + args := flag.Args() + logrus.Debugln("输入参数为", args) + + if len(args) == 0 { + logrus.Fatalln("请输入需要下载的包名") + } + var pkg = args[0] + var tag string + if len(args) > 1 { + tag = args[1] + } else { + var found bool + pkg, tag, found = strings.Cut(pkg, ":") + if !found { + tag = "latest" + } + } + err := dget.Install(pkg, tag, *arch, *printInfo) + if err != nil { + logrus.Fatalln("下载发生错误", err) + } + os.Exit(0) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..9400d66 --- /dev/null +++ b/go.mod @@ -0,0 +1,11 @@ +module gitee.com/extrame/dget + +go 1.15 + +require ( + github.com/boltdb/bolt v1.3.1 + github.com/docker/distribution v2.8.1+incompatible // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.0.2 // indirect + github.com/sirupsen/logrus v1.8.1 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..a957484 --- /dev/null +++ b/go.sum @@ -0,0 +1,15 @@ +github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= +github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= +github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/install.go b/install.go new file mode 100644 index 0000000..341ae70 --- /dev/null +++ b/install.go @@ -0,0 +1,409 @@ +package dget + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/docker/distribution/manifest/manifestlist" + "github.com/opencontainers/go-digest" + "github.com/sirupsen/logrus" +) + +const _registry = "registry-1.docker.io" +const _authUrl = "https://auth.docker.io/token" +const _regService = "registry.docker.io" + +type LayerInfo struct { + Id string `json:"id"` + Parent string `json:"parent"` + Created time.Time `json:"created"` + ContainerConfig struct { + Hostname string + Domainname string + User string + AttachStdin bool + AttachStdout bool + AttachStderr bool + Tty bool + OpenStdin bool + StdinOnce bool + Env []string + CMd []string + Image string + Volumes map[string]interface{} + WorkingDir string + Entrypoint []string + OnBuild []string + Labels map[string]interface{} + } `json:"container_config"` +} + +type Layer struct { + Digest string + Urls []string +} + +type Info struct { + Layers []Layer `json:"layers"` + Config struct { + Digest digest.Digest `json:"digest,omitempty"` + } `json:"config"` +} + +type PackageConfig struct { + Config string + RepoTags []string + Layers []string +} + +func Install(d, tag string, arch string, printInfo bool) (err error) { + var authUrl = _authUrl + var regService = _regService + resp, err := http.Get(fmt.Sprintf("https://%s/v2/", _registry)) + if err == nil { + if !strings.Contains(d, "/") { + d = "library/" + d + } + if resp.StatusCode == 401 { + //Bearer realm="https://auth.docker.io/token",service="registry.docker.io" + var hAuths = strings.Split(resp.Header.Get("Www-Authenticate"), "\"") + if len(hAuths) > 1 { + authUrl = hAuths[1] + } + if len(hAuths) > 3 { + regService = hAuths[3] + } else { + regService = "" + } + } + resp.Body.Close() + var accessToken string + logrus.Debugln("reg_service", regService) + + accessToken, err = getAuthHead("application/vnd.docker.distribution.manifest.v2+json", authUrl, regService, d) + if err == nil { + + var req *http.Request + + var url = fmt.Sprintf("https://%s/v2/%s/manifests/%s", _registry, d, tag) + req, err = http.NewRequest("GET", url, nil) + logrus.Infoln("获取manifests信息", url) + if err == nil { + logrus.Debugln("Authorization by", accessToken) + req.Header.Add("Authorization", "Bearer "+accessToken) + req.Header.Add("Accept", "application/vnd.docker.distribution.manifest.list.v2+json") + + var authHeader = req.Header + + resp, err = http.DefaultClient.Do(req) + if resp.StatusCode != 200 { + bts, er := ioutil.ReadAll(resp.Body) + resp.Body.Close() + logrus.Debugln(string(bts), er) + switch resp.StatusCode { + case 401: + logrus.Errorf("[-] Cannot fetch manifest for %s [HTTP %d] with error access_token", d, resp.StatusCode) + case 404: + logrus.Errorf("[-] Cannot fetch manifest for %s [HTTP %d] with url %s", d, resp.StatusCode, url) + resp.Body.Close() + req.Header.Set("Accept", "application/vnd.docker.distribution.manifest.list.v2+json") + resp, err = http.DefaultClient.Do(req) + bts, er := ioutil.ReadAll(resp.Body) + fmt.Println(string(bts), er) + } + //TODO + + os.Exit(1) + } else { + var info manifestlist.ManifestList + var bts []byte + bts, err = io.ReadAll(resp.Body) + + if err == nil { + + err = json.Unmarshal(bts, &info) + + if err == nil { + resp.Body.Close() + + logrus.Infof("获得%d个架构信息:", len(info.Manifests)) + + var selectedManifest *manifestlist.ManifestDescriptor + for i := 0; i < len(info.Manifests); i++ { + var m = info.Manifests[i] + logrus.Infof("[%d]架构:%s,OS:%s", i+1, m.Platform.Architecture, m.Platform.OS) + if m.Platform.OS+"/"+m.Platform.Architecture == arch { + logrus.Infoln("找到匹配的架构,开始下载") + selectedManifest = &m + } + } + if printInfo { + fmt.Println(string(bts)) + os.Exit(0) + } + + if selectedManifest == nil { + return errors.New("未找到匹配的架构:" + arch) + } + + req.Header.Set("Accept", selectedManifest.MediaType) + + resp, err = http.DefaultClient.Do(req) + var info Info + err = json.NewDecoder(resp.Body).Decode(&info) + + if err == nil { + resp.Body.Close() + logrus.Infof("获得Manifest信息,共%d层需要下载", len(info.Layers)) + + var tmpDir = fmt.Sprintf("tmp_%s_%s", d, tag) + err = os.MkdirAll(tmpDir, 0777) + if err == nil { + if _, e := os.Stat(filepath.Join(tmpDir, "repositories")); e == nil { + logrus.Info(tmpDir, "is downloaded,use dir as cache") + } else { + req, err = http.NewRequest("GET", fmt.Sprintf("https://%s/v2/%s/blobs/%s", _registry, d, info.Config.Digest), nil) + if err == nil { + req.Header = authHeader + resp, err = http.DefaultClient.Do(req) + if err == nil { + var dest *os.File + dest, err = os.OpenFile(filepath.Join(tmpDir, info.Config.Digest.Encoded()+".json"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) + if err == nil { + var bts []byte + bts, err = ioutil.ReadAll(resp.Body) + var lastLayerInfo LayerInfo + err = json.Unmarshal(bts, &lastLayerInfo) + resp.Body.Close() + + var config []PackageConfig + config = append(config, PackageConfig{ + Config: info.Config.Digest.Encoded() + ".json", + RepoTags: []string{d + ":" + tag}, + }) + if err == nil { + _, err = io.Copy(dest, bytes.NewReader(bts)) + dest.Close() + if err == nil { + parentid := "" + var fakeLayerId string + for n, layer := range info.Layers { + namer := sha256.New() + namer.Write([]byte(parentid + "\n" + layer.Digest + "\n")) + fakeLayerId = hex.EncodeToString(namer.Sum(nil)) + logrus.Infoln("handle layer", n, fakeLayerId, layer.Urls) + layerDirName := filepath.Join(tmpDir, fakeLayerId) + err = os.Mkdir(layerDirName, 0777) + if _, er := os.Stat(filepath.Join(layerDirName, "layer.tar")); er == nil { + logrus.Infoln("layer", fakeLayerId, "is existed, continue") + config[0].Layers = append(config[0].Layers, fakeLayerId+"/layer.tar") + parentid = fakeLayerId + continue + } + if err == nil || os.IsExist(err) { + err = ioutil.WriteFile(filepath.Join(layerDirName, "VERSION"), []byte("1.0"), 0666) + if err == nil { + req, err = http.NewRequest("GET", fmt.Sprintf("https://%s/v2/%s/blobs/%s", _registry, d, layer.Digest), nil) + if err == nil { + req.Header = authHeader + req.Header.Set("Accept", "application/vnd.docker.distribution.manifest.v2+json") + resp, err = http.DefaultClient.Do(req) + if err == nil { + if resp.StatusCode != 200 { + defer resp.Body.Close() + if len(layer.Urls) > 0 { + req, err = http.NewRequest("GET", layer.Urls[0], nil) + if err == nil { + req.Header = authHeader + req.Header.Set("Accept", "application/vnd.docker.distribution.manifest.v2+json") + resp, err = http.DefaultClient.Do(req) + if err == nil { + if resp.StatusCode != 200 { + err = fmt.Errorf("download from customized url fail for layer[%d]", n) + goto response + } + } + } + } else { + bts, _ := ioutil.ReadAll(resp.Body) + logrus.Fatalln("下载失败", string(bts)) + } + } + } + if err != nil { + logrus.Errorf("请求第%d/%d层失败:%v", n+1, len(info.Layers), err) + } else { + logrus.Infof("请求第%d/%d层成功", n+1, len(info.Layers)) + } + var dst *os.File + dst, err = os.OpenFile(filepath.Join(layerDirName, "layer.tar.part"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) + if err == nil { + var greader *gzip.Reader + greader, err = gzip.NewReader(resp.Body) + if err == nil { + _, err = io.Copy(dst, greader) + if err == nil { + dst.Close() + var layerInfo LayerInfo + if n == len(info.Layers)-1 { + layerInfo = lastLayerInfo + } + layerInfo.Id = fakeLayerId + if parentid != "" { + layerInfo.Parent = parentid + } + parentid = fakeLayerId + var jsonFile *os.File + jsonFile, err = os.OpenFile(filepath.Join(layerDirName, "json"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) + if err == nil { + err = json.NewEncoder(jsonFile).Encode(&layerInfo) + if err == nil { + jsonFile.Close() + err = os.Rename(filepath.Join(layerDirName, "layer.tar.part"), filepath.Join(layerDirName, "layer.tar")) + } + } + } + } + } + if err != nil { + logrus.Errorf("保存第%d/%d层失败,%v", n+1, len(info.Layers), err) + } else { + logrus.Infof("保存第%d/%d层成功", n+1, len(info.Layers)) + } + if err != nil { + goto response + } else { + config[0].Layers = append(config[0].Layers, fakeLayerId+"/layer.tar") + } + } + } + } + } + var manifest *os.File + manifest, err = os.OpenFile(filepath.Join(tmpDir, "manifest.json"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) + if err == nil { + err = json.NewEncoder(manifest).Encode(&config) + if err == nil { + manifest.Close() + var repositories = make(map[string]interface{}) + repositories[d] = map[string]string{ + tag: fakeLayerId, + } + var rFile *os.File + rFile, err = os.OpenFile(filepath.Join(tmpDir, "repositories"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) + if err == nil { + err = json.NewEncoder(rFile).Encode(&repositories) + goto maketar + } + } + + } + } + } + } + } + } + + } + maketar: + if err == nil { + err = writeDirToTarGz(tmpDir, tmpDir+"-img.tar.gz") + if err == nil { + fmt.Println("write tar success", tmpDir+"-img.tar.gz") + } + } + } + } + } + } + } + } + } + } +response: + return +} + +func getAuthHead(u, a, r, d string) (string, error) { + resp, err := http.Get(fmt.Sprintf("%s?service=%s&scope=repository:%s:pull", a, r, d)) + defer resp.Body.Close() + if err == nil { + var results map[string]interface{} + err = json.NewDecoder(resp.Body).Decode(&results) + logrus.Debug(results) + if err == nil { + return results["access_token"].(string), nil + } + } + return "", err +} + +func writeDirToTarGz(sourcedir, destinationfile string) error { + // create tar file + gzFile, err := os.Create(destinationfile) + gf := gzip.NewWriter(gzFile) + tw := tar.NewWriter(gf) + if err == nil { + + defer func() { + tw.Close() + gf.Close() + gzFile.Close() + }() + + // get list of files + return filepath.Walk(sourcedir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + relPath, err := filepath.Rel(sourcedir, path) + if err == nil && relPath != "." { + header, err := tar.FileInfoHeader(info, path) + if err != nil { + return err + } + + // must provide real name + // (see https://golang.org/src/archive/tar/common.go?#L626) + header.Name = filepath.ToSlash(relPath) + + // write header + if err := tw.WriteHeader(header); err != nil { + return err + } + // if not a dir, write file content + if !info.IsDir() { + data, err := os.Open(path) + if err != nil { + return err + } + if _, err := io.Copy(tw, data); err != nil { + return err + } + } + return nil + } + return err + }) + + } + return err +} + +func SetLogLevel(lvl logrus.Level) { + logrus.SetLevel(lvl) + logrus.Debugln("设置日志级别为", lvl) +} diff --git a/stat.go b/stat.go new file mode 100644 index 0000000..18eb61e --- /dev/null +++ b/stat.go @@ -0,0 +1,19 @@ +package dget + +import ( + "os" + "path/filepath" + + "github.com/boltdb/bolt" +) + +var cache *bolt.DB + +func initCache() error { + db, err := bolt.Open(filepath.Join(os.TempDir(), "fb-runner.db"), 0600, nil) + if err != nil { + return err + } + cache = db + return nil +}