deno.land / x / esm@v135_2 / server / npm.go
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508package server
import ( "encoding/base64" "encoding/json" "fmt" "io" "net/http" "os" "os/exec" "path" "sort" "strings" "sync" "time"
"github.com/esm-dev/esm.sh/server/storage"
"github.com/Masterminds/semver/v3" "github.com/ije/gox/utils" "github.com/ije/gox/valid")
// ref https://github.com/npm/validate-npm-package-namevar npmNaming = valid.Validator{valid.FromTo{'a', 'z'}, valid.FromTo{'A', 'Z'}, valid.FromTo{'0', '9'}, valid.Eq('.'), valid.Eq('-'), valid.Eq('_')}
// NpmPackageVerions defines versions of a NPM packagetype NpmPackageVerions struct { DistTags map[string]string `json:"dist-tags"` Versions map[string]NpmPackageInfo `json:"versions"`}
// NpmPackageJSON defines the package.json of NPMtype NpmPackageJSON struct { Name string `json:"name"` Version string `json:"version"` Type string `json:"type,omitempty"` Main string `json:"main,omitempty"` Browser StringOrMap `json:"browser,omitempty"` Module StringOrMap `json:"module,omitempty"` ES2015 StringOrMap `json:"es2015,omitempty"` JsNextMain string `json:"jsnext:main,omitempty"` Types string `json:"types,omitempty"` Typings string `json:"typings,omitempty"` SideEffects interface{} `json:"sideEffects,omitempty"` Dependencies map[string]string `json:"dependencies,omitempty"` PeerDependencies map[string]string `json:"peerDependencies,omitempty"` Imports map[string]interface{} `json:"imports,omitempty"` TypesVersions map[string]interface{} `json:"typesVersions,omitempty"` PkgExports json.RawMessage `json:"exports,omitempty"` Deprecated interface{} `json:"deprecated,omitempty"` ESMConfig interface{} `json:"esm.sh,omitempty"`}
func (a *NpmPackageJSON) ToNpmPackage() *NpmPackageInfo { browser := map[string]string{} if a.Browser.Str != "" { browser["."] = a.Browser.Str } if a.Browser.Map != nil { for k, v := range a.Browser.Map { s, isStr := v.(string) if isStr { browser[k] = s } else { b, ok := v.(bool) if ok && !b { browser[k] = "" } } } } deprecated := "" if a.Deprecated != nil { if s, ok := a.Deprecated.(string); ok { deprecated = s } } esmConfig := map[string]interface{}{} if a.ESMConfig != nil { if v, ok := a.ESMConfig.(map[string]interface{}); ok { esmConfig = v } } var sideEffects *stringSet = nil sideEffectsFalse := false if a.SideEffects != nil { if s, ok := a.SideEffects.(string); ok { sideEffectsFalse = s == "false" } else if b, ok := a.SideEffects.(bool); ok { sideEffectsFalse = !b } else if m, ok := a.SideEffects.([]interface{}); ok && len(m) > 0 { sideEffects = newStringSet() for _, v := range m { if name, ok := v.(string); ok { sideEffects.Add(name) } } } } var pkgExports interface{} = nil if rawExports := a.PkgExports; rawExports != nil { var v interface{} if json.Unmarshal(rawExports, &v) == nil { if s, ok := v.(string); ok { if len(s) > 0 { pkgExports = s } } else if _, ok := v.(map[string]interface{}); ok { om := newOrderedMap() if om.UnmarshalJSON(rawExports) == nil { pkgExports = om } else { pkgExports = v } } } } return &NpmPackageInfo{ Name: a.Name, Version: a.Version, Type: a.Type, Main: a.Main, Module: a.Module.MainValue(), ES2015: a.ES2015.MainValue(), JsNextMain: a.JsNextMain, Types: a.Types, Typings: a.Typings, Browser: browser, SideEffectsFalse: sideEffectsFalse, SideEffects: sideEffects, Dependencies: a.Dependencies, PeerDependencies: a.PeerDependencies, Imports: a.Imports, TypesVersions: a.TypesVersions, PkgExports: pkgExports, Deprecated: deprecated, ESMConfig: esmConfig, }}
// NpmPackage defines the package.jsontype NpmPackageInfo struct { Name string Version string Type string Main string Module string ES2015 string JsNextMain string Types string Typings string SideEffectsFalse bool SideEffects *stringSet Browser map[string]string Dependencies map[string]string PeerDependencies map[string]string Imports map[string]interface{} TypesVersions map[string]interface{} PkgExports interface{} Deprecated string ESMConfig map[string]interface{}}
func (a *NpmPackageInfo) UnmarshalJSON(b []byte) error { var n NpmPackageJSON if err := json.Unmarshal(b, &n); err != nil { return err } *a = *n.ToNpmPackage() return nil}
func getPackageInfo(wd string, name string, version string) (info NpmPackageInfo, fromPackageJSON bool, err error) { if name == "@types/node" { info = NpmPackageInfo{ Name: "@types/node", Version: nodeTypesVersion, Types: "index.d.ts", } return }
if wd != "" { pkgJsonPath := path.Join(wd, "node_modules", name, "package.json") if fileExists(pkgJsonPath) && utils.ParseJSONFile(pkgJsonPath, &info) == nil { info, err = fixPkgVersion(info) fromPackageJSON = true return } }
info, err = fetchPackageInfo(name, version) if err == nil { info, err = fixPkgVersion(info) } return}
func fetchPackageInfo(name string, version string) (info NpmPackageInfo, err error) { a := strings.Split(strings.Trim(name, "/"), "/") name = a[0] if strings.HasPrefix(name, "@") && len(a) > 1 { name = a[0] + "/" + a[1] }
if strings.HasPrefix(version, "=") || strings.HasPrefix(version, "v") { version = version[1:] } if version == "" { version = "latest" } isFullVersion := regexpFullVersion.MatchString(version)
cacheKey := fmt.Sprintf("npm:%s@%s", name, version) lock := getFetchLock(cacheKey) lock.Lock() defer lock.Unlock()
// check cache firstly if cache != nil { var data []byte data, err = cache.Get(cacheKey) if err == nil && json.Unmarshal(data, &info) == nil { return } if err != nil && err != storage.ErrNotFound && err != storage.ErrExpired { log.Error("cache:", err) } }
start := time.Now() defer func() { if err == nil { log.Debugf("lookup package(%s@%s) in %v", name, info.Version, time.Since(start)) } }()
isJsrScope := strings.HasPrefix(name, "@jsr/") url := cfg.NpmRegistry + name if isJsrScope { url = "https://npm.jsr.io/" + name } else if cfg.NpmRegistryScope != "" { isInScope := strings.HasPrefix(name, cfg.NpmRegistryScope) if !isInScope { url = "https://registry.npmjs.org/" + name } }
if isFullVersion && !isJsrScope { url += "/" + version } req, err := http.NewRequest("GET", url, nil) if err != nil { return } if cfg.NpmToken != "" && !isJsrScope { req.Header.Set("Authorization", "Bearer "+cfg.NpmToken) } if cfg.NpmUser != "" && cfg.NpmPassword != "" && !isJsrScope { req.SetBasicAuth(cfg.NpmUser, cfg.NpmPassword) }
resp, err := httpClient.Do(req) if err != nil { return } defer resp.Body.Close()
if resp.StatusCode == 404 || resp.StatusCode == 401 { if isFullVersion { err = fmt.Errorf("npm: version %s of '%s' not found", version, name) } else { err = fmt.Errorf("npm: package '%s' not found", name) } return }
if resp.StatusCode != 200 { ret, _ := io.ReadAll(resp.Body) err = fmt.Errorf("npm: could not get metadata of package '%s' (%s: %s)", name, resp.Status, string(ret)) return }
if isFullVersion && !isJsrScope { err = json.NewDecoder(resp.Body).Decode(&info) if err != nil { return } if cache != nil { cache.Set(cacheKey, utils.MustEncodeJSON(info), 7*24*time.Hour) } return }
var h NpmPackageVerions err = json.NewDecoder(resp.Body).Decode(&h) if err != nil { return }
if len(h.Versions) == 0 { err = fmt.Errorf("npm: missing `versions` field") return }
distVersion, ok := h.DistTags[version] if ok { info = h.Versions[distVersion] } else { var c *semver.Constraints c, err = semver.NewConstraint(version) if err != nil && version != "latest" { return fetchPackageInfo(name, "latest") } vs := make([]*semver.Version, len(h.Versions)) i := 0 for v := range h.Versions { // ignore prerelease versions if !strings.ContainsRune(version, '-') && strings.ContainsRune(v, '-') { continue } var ver *semver.Version ver, err = semver.NewVersion(v) if err != nil { return } if c.Check(ver) { vs[i] = ver i++ } } if i > 0 { vs = vs[:i] if i > 1 { sort.Sort(semver.Collection(vs)) } info = h.Versions[vs[i-1].String()] } }
if info.Version == "" { err = fmt.Errorf("npm: version %s of '%s' not found", version, name) return }
// cache package info for 10 minutes if cache != nil { cache.Set(cacheKey, utils.MustEncodeJSON(info), 10*time.Minute) } return}
func installPackage(wd string, pkg Pkg) (err error) { pkgVersionName := pkg.VersionName()
// only one install process allowed at the same time lock := getInstallLock(pkgVersionName) lock.Lock() defer lock.Unlock()
// ensure package.json file to prevent read up-levels packageFilePath := path.Join(wd, "package.json") if pkg.FromEsmsh { err = copyRawBuildFile(pkg.Name, "package.json", wd) } else if pkg.FromGithub || !fileExists(packageFilePath) { fileContent := []byte("{}") if pkg.FromGithub { fileContent = []byte(fmt.Sprintf( `{"dependencies": {"%s": "%s"}}`, pkg.Name, fmt.Sprintf("git+https://github.com/%s.git#%s", pkg.Name, pkg.Version), )) } ensureDir(wd) err = os.WriteFile(packageFilePath, fileContent, 0644) } if err != nil { return fmt.Errorf("ensure package.json failed: %s", pkgVersionName) }
for i := 0; i < 3; i++ { if pkg.FromEsmsh { err = pnpmInstall(wd) if err == nil { installDir := path.Join(wd, "node_modules", pkg.Name) for _, name := range []string{"package.json", "index.mjs", "index.d.ts"} { err = copyRawBuildFile(pkg.Name, name, installDir) if err != nil { break } } } } else if pkg.FromGithub { err = pnpmInstall(wd) // pnpm will ignore github package which has been installed without `package.json` file if err == nil && !dirExists(path.Join(wd, "node_modules", pkg.Name)) { err = ghInstall(wd, pkg.Name, pkg.Version) } } else if regexpFullVersion.MatchString(pkg.Version) { err = pnpmInstall(wd, pkgVersionName, "--prefer-offline") } else { err = pnpmInstall(wd, pkgVersionName) } packageFilePath := path.Join(wd, "node_modules", pkg.Name, "package.json") if err == nil && !fileExists(packageFilePath) { if pkg.FromGithub { ensureDir(path.Dir(packageFilePath)) err = os.WriteFile(packageFilePath, utils.MustEncodeJSON(pkg), 0644) } else { err = fmt.Errorf("pnpm install %s: package.json not found", pkg) } } if err == nil { break } if i < 2 { time.Sleep(100 * time.Millisecond) } } return}
func pnpmInstall(wd string, packages ...string) (err error) { var args []string if len(packages) > 0 { args = append([]string{"add"}, packages...) } else { args = []string{"install"} } args = append( args, "--ignore-scripts", "--loglevel", "error", ) start := time.Now() cmd := exec.Command("pnpm", args...) cmd.Dir = wd if cfg.NpmToken != "" { cmd.Env = append(os.Environ(), "ESM_NPM_TOKEN="+cfg.NpmToken) } if cfg.NpmUser != "" && cfg.NpmPassword != "" { data := []byte(cfg.NpmPassword) password := make([]byte, base64.StdEncoding.EncodedLen(len(data))) base64.StdEncoding.Encode(password, data) cmd.Env = append( os.Environ(), "ESM_NPM_USER="+cfg.NpmUser, "ESM_NPM_PASSWORD="+string(password), ) } output, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("pnpm add %s: %s", strings.Join(packages, ","), string(output)) } if len(packages) > 0 { log.Debug("pnpm add", strings.Join(packages, ","), "in", time.Since(start)) } else { log.Debug("pnpm install in", time.Since(start)) } return}
// ref https://github.com/npm/validate-npm-package-namefunc validatePackageName(name string) bool { scope := "" nameWithoutScope := name if strings.HasPrefix(name, "@") { scope, nameWithoutScope = utils.SplitByFirstByte(name, '/') scope = scope[1:] } if (scope != "" && !npmNaming.Is(scope)) || (nameWithoutScope == "" || !npmNaming.Is(nameWithoutScope)) || len(name) > 214 { return false } return true}
// added by @jimisaacsfunc toTypesPackageName(pkgName string) string { if strings.HasPrefix(pkgName, "@") { pkgName = strings.Replace(pkgName[1:], "/", "__", 1) } return "@types/" + pkgName}
func fixPkgVersion(info NpmPackageInfo) (NpmPackageInfo, error) { for prefix, ver := range fixedPkgVersions { if strings.HasPrefix(info.Name+"@"+info.Version, prefix) { return fetchPackageInfo(info.Name, ver) } } return info, nil}
func isTypesOnlyPackage(p NpmPackageInfo) bool { return p.Main == "" && p.Module == "" && p.Types != ""}
func getInstallLock(key string) *sync.Mutex { v, _ := installLocks.LoadOrStore(key, &sync.Mutex{}) return v.(*sync.Mutex)}
func getFetchLock(key string) *sync.Mutex { v, _ := fetchLocks.LoadOrStore(key, &sync.Mutex{}) return v.(*sync.Mutex)}
Version Info