using System; using System.Collections.Generic; using System.IO; using System.IO.Compression; using System.Linq; using System.Text; using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Options; using isn.abst; using isn.Abstract; using isnd.Data; using isnd.Data.Catalog; using isnd.Data.Packages; using isnd.Entities; using isnd.Helpers; using isnd.Interfaces; using isnd.ViewModels; using NuGet.Packaging.Core; using NuGet.Versioning; using System.Xml; using System.Xml.Linq; using System.Threading; using NuGet.Protocol; namespace isnd.Services { public class PackageManager : IPackageManager { public const string BASE_API_LEVEL = "3.0.0"; readonly ApplicationDbContext dbContext; public PackageManager(ApplicationDbContext dbContext, IOptions siteConfigOptionsOptions) { this.dbContext = dbContext; isndSettings = siteConfigOptionsOptions.Value; apiBase = isndSettings.ExternalUrl + Constants.ApiVersionPrefix; } public IEnumerable GetResources() { var res = new List { new Resource(apiBase + ApiConfig.Package, "PackagePublish/2.0.0") { Comment = "Package Publish service" }, // under dev, only leash in release mode new Resource(apiBase + ApiConfig.Package + "/{id}/{version}", "PackageDetailsUriTemplate/5.1.0") { Comment = "URI template used by NuGet Client to construct details URL for packages" }, new Resource(apiBase + ApiConfig.Content, "PackageBaseAddress/3.0.0") { Comment = "Package Base Address service - " + "Base URL of where NuGet packages are stored, in the format " // "https:///nupkg/{id-lower}/{version-lower}/{id-lower}.{version-lower}.nupkg" }, new Resource(apiBase + ApiConfig.AutoComplete, "SearchAutocompleteService/" + BASE_API_LEVEL) { Comment = "Auto complete service" }, new Resource(apiBase + ApiConfig.Search, "SearchQueryService") { Comment = "Query endpoint of NuGet Search service (primary) used by RC clients" }, new Resource(apiBase + ApiConfig.Search, "SearchQueryService/3.0.0-beta") { Comment = "Query endpoint of NuGet Search service (primary) used by RC clients" }, new Resource(apiBase + ApiConfig.Search, "SearchQueryService/3.0.0-rc") { Comment = "Query endpoint of NuGet Search service (primary) used by RC clients" }, new Resource(apiBase + ApiConfig.Search, "SearchQueryService/3.5.0") { Comment = "Query endpoint of NuGet Search service (primary) used by RC clients" }, new Resource(apiBase + ApiConfig.Registration, "RegistrationsBaseUrl/Versioned") { Comment = "Base URL of storage where package registration info is stored. " + "This base URL includes SemVer 2.0.0 packages." } }; return res; } public AutoCompleteResult AutoComplete(string id, int skip, int take, bool prerelease = false, string packageType = null) { var scope = dbContext.PackageVersions.Where( v => v.PackageId.StartsWith(id) && (prerelease || !v.IsPrerelease) && (packageType == null || v.Type == packageType) ) .OrderBy(v => v.FullString); return new AutoCompleteResult { totalHits = scope.Count(), data = scope.Select(v => v.FullString) .Skip(skip).Take(take).ToArray() }; } public string[] GetVersions( string id, NuGetVersion parsedVersion = null, bool prerelease = false, string packageType = null, int skip = 0, int take = 25) { return dbContext.PackageVersions.Where( v => v.PackageId == id && (prerelease || !v.IsPrerelease) && (packageType == null || v.Type == packageType) && (parsedVersion == null || parsedVersion.CompareTo (new SemanticVersion(v.Major, v.Minor, v.Patch)) < 0) ) .OrderBy(v => v.NugetVersion) .Select(v => v.FullString) .Skip(skip).Take(take).ToArray(); } public string CatalogBaseUrl => apiBase; private IsndSettings isndSettings; private readonly string apiBase; public virtual async Task GetCatalogIndexAsync() { return await UpdateCatalogForAsync(null); } public async Task UpdateCatalogForAsync (Commit reason = null) { int i = 0; string baseId = apiBase + ApiConfig.Catalog; string baseRegistrationId = $"{apiBase}{ApiConfig.Registration}"; PackageRegistration index = new PackageRegistration(baseId); var scope = await dbContext.Commits.OrderBy(c => c.TimeStamp).ToArrayAsync(); i = isndSettings.CatalogPageLen; foreach (var commit in scope) { var validPkgs = (await dbContext.Packages .Include(po => po.Owner) .Include(pkg => pkg.Versions) .Include(pkg => pkg.LatestCommit) .ToArrayAsync()) .GroupBy((q) => q.Id); foreach (var pkgIdGroup in validPkgs) { RegistrationPage page = index.Items.FirstOrDefault (p => p.GetPackageId() == pkgIdGroup.Key); if (page == null) { page = new RegistrationPage(pkgIdGroup.Key, apiBase); index.Items.Add(page); } foreach (var package in pkgIdGroup) { page.AddVersionRange(package.Versions); } } reason = commit; i++; } return index; } public async Task DeletePackageAsync (string pkgId, string version, string type) { // TODO deletion on disk var commit = new Commit { Action = PackageAction.DeletePackage, TimeStamp = DateTimeOffset.Now.ToUniversalTime() }; dbContext.Commits.Add(commit); var pkg = await dbContext.PackageVersions.SingleOrDefaultAsync( v => v.PackageId == pkgId && v.FullString == version && v.Type == type ); if (pkg == null) { return new PackageDeletionReport { Deleted = false }; } dbContext.PackageVersions.Remove(pkg); await dbContext.SaveChangesAsync(); await UpdateCatalogForAsync(commit); return new PackageDeletionReport { Deleted = true, DeletedVersion = pkg }; } public async Task GetPackageAsync (string pkgId, string version, string type) { return await dbContext.PackageVersions.SingleOrDefaultAsync( v => v.PackageId == pkgId && v.FullString == version && (type==null || v.Type == type) ); } public async Task GetCatalogEntryAsync (string pkgId, string semver, string pkgType = "Dependency") { var version = await dbContext.PackageVersions .Include(v => v.Package) .Include(v => v.Package.LatestCommit) .Include(v => v.Package.Owner) .Include(v => v.DependencyGroups) .Include(v => v.LatestCommit) .Where(v => v.PackageId == pkgId && v.FullString == semver && v.LatestCommit != null && (pkgType == null || pkgType == v.Type) ).SingleOrDefaultAsync(); foreach (var g in version.DependencyGroups) { g.Dependencies = dbContext.Dependencies.Where(d => d.DependencyGroupId == g.Id).ToList(); } return version.ToPackage(apiBase); } public async Task UserAskForPackageDeletionAsync (string uid, string id, string lower, string type) { PackageVersion packageVersion = await dbContext.PackageVersions .Include(pv => pv.Package) .FirstOrDefaultAsync(m => m.PackageId == id && m.FullString == lower && (type==null || m.Type == type)); if (packageVersion == null) return null; if (packageVersion.Package.OwnerId != uid) return null; return new PackageDeletionReport { Deleted = true, DeletedVersion = packageVersion }; } public IEnumerable SearchCatalogEntriesById (string pkgId, string semver, string pkgType, bool preRelease) { // PackageDependency return dbContext.PackageVersions .Include(v => v.Package) .Include(v => v.Package.Owner) .Include(v => v.Package.LatestCommit) .Include(v => v.LatestCommit) .Include(v => v.DependencyGroups) .Where(v => v.PackageId == pkgId && semver == v.FullString && (pkgType == null || pkgType == v.Type) && (preRelease || !v.IsPrerelease)) .OrderByDescending(p => p.CommitNId) .Select(p => p.ToPackage(apiBase)) ; } public PackageVersion GetPackage(string pkgId, string semver, string pkgType) { return dbContext.PackageVersions .Include(v => v.Package) .Include(v => v.LatestCommit) .Include(v => v.DependencyGroups.Last().Dependencies) .Single(v => v.PackageId == pkgId && semver == v.FullString && (pkgType == null || pkgType == v.Type)); } public async Task GetPackageRegistrationIndexAsync (PackageRegistrationQuery query) { if (string.IsNullOrWhiteSpace(query.Query)) return null; query.Query = query.Query.ToLower(); var scope = await dbContext.PackageVersions .Include(p => p.Package) .Include(p => p.Package.Owner) .Include(p => p.LatestCommit) .Where(p => p.PackageId.ToLower() == query.Query).ToArrayAsync(); if (scope == null) return null; if (scope.Length == 0) return null; string bid = $"{apiBase}{ApiConfig.Registration}"; foreach (var version in scope) { version.DependencyGroups = dbContext.PackageDependencyGroups.Include(d => d.Dependencies) .Where(d => d.PackageId == version.PackageId && d.PackageVersionFullString == version.FullString) .ToList(); version.LatestCommit = dbContext.Commits.Single(c => c.Id == version.CommitNId); } return new PackageRegistration(apiBase, query.Query, scope); } public async Task SearchPackageAsync(PackageRegistrationQuery query) { string bid = $"{apiBase}{ApiConfig.Registration}"; if (string.IsNullOrWhiteSpace(query.Query)) query.Query = ""; var packages = await dbContext.Packages .Include(g => g.Versions).OrderBy(v=>v.CommitNId) .Where(d => d.Id.StartsWith(query.Query) && (query.Prerelease || d.Versions.Any(v => !v.IsPrerelease))) .Where(p=>p.Versions.Count>=0) .Skip(query.Skip).Take(query.Take).ToArrayAsync(); foreach (var package in packages) foreach (var version in package.Versions) { version.DependencyGroups = dbContext.PackageDependencyGroups.Include(d => d.Dependencies) .Where(d => d.PackageVersionFullString == version.FullString && d.PackageId == version.PackageId) .ToList(); } return new PackageSearchResult(packages, apiBase, packages.Count()); } public async Task PutPackageAsync(Stream packageStream, string ownerId) { PackageVersion version = null; using (packageStream) { using (var archive = new ZipArchive(packageStream)) { var spec = archive.Entries.FirstOrDefault(e => e.FullName.EndsWith("." + Constants.SpecFileExtension)); if (spec == null) throw new InvalidPackageException("no " + Constants.SpecFileExtension + " from archive"); string pkgPath; NuGetVersion nugetVersion; string pkgId; string fullPath; using var specificationStream = spec.Open(); using XmlReader xmlReader = XmlReader.Create(specificationStream); var xMeta = XElement.Load(xmlReader, LoadOptions.None).Descendants().First(); string packageDescription = xMeta.Descendants().FirstOrDefault(x => x.Name.LocalName == "description")?.Value; var dependencies = xMeta .Descendants().FirstOrDefault(x => x.Name.LocalName == "dependencies"); var frameworkReferences= xMeta .Descendants().FirstOrDefault(x => x.Name.LocalName == "frameworkReferences"); var frameWorks = (dependencies ?? frameworkReferences) .Descendants().Where(x => x.Name.LocalName == "group") .Select(x => NewFrameworkDependencyGroup(x)).ToArray(); // FIXME default package type or null var types = "Dependency"; pkgId = xMeta.Descendants().FirstOrDefault(x => x.Name.LocalName == "id")?.Value; string pkgVersion = xMeta.Descendants().FirstOrDefault(x => x.Name.LocalName == "version")?.Value; if (!NuGetVersion.TryParse(pkgVersion, out nugetVersion)) throw new InvalidPackageException("metadata/version"); string packageIdPath = Path.Combine(isndSettings.PackagesRootDir, pkgId); pkgPath = Path.Combine(packageIdPath, nugetVersion.ToFullString()); string name = $"{pkgId}-{nugetVersion}." + Constants.PacketFileExtension; fullPath = Path.Combine(pkgPath, name); var packageIdPathInfo = new DirectoryInfo(packageIdPath); Data.Packages.Package pkg = dbContext.Packages.SingleOrDefault(p => p.Id == pkgId); Commit commit = new Commit { Action = PackageAction.PublishPackage, TimeStamp = DateTimeOffset.Now.ToUniversalTime() }; if (pkg != null) { // Update } else { // First version pkg = new Package { Id = pkgId, OwnerId = ownerId, }; dbContext.Packages.Add(pkg); } pkg.Public = true; pkg.LatestCommit = commit; pkg.Description = packageDescription; // here, the package is or new, or owned by the key owner if (!packageIdPathInfo.Exists) packageIdPathInfo.Create(); var dest = new FileInfo(fullPath); var destDir = new DirectoryInfo(dest.DirectoryName); if (dest.Exists) dest.Delete(); if (!destDir.Exists) destDir.Create(); packageStream.Seek(0, SeekOrigin.Begin); using (var fileStream = File.Create(fullPath)) { await packageStream.CopyToAsync(fileStream); } string fullStringVersion = nugetVersion.ToFullString(); var pkgVersions = dbContext.PackageVersions.Where (v => v.PackageId == pkg.Id && v.FullString == fullStringVersion); if (pkgVersions.Count() > 0) { foreach (var v in pkgVersions.ToArray()) dbContext.PackageVersions.Remove(v); } string versionFullString = nugetVersion.ToFullString(); dbContext.PackageVersions.Add (version = new PackageVersion { Package = pkg, Major = nugetVersion.Major, Minor = nugetVersion.Minor, Patch = nugetVersion.Patch, Revision = nugetVersion.Revision, IsPrerelease = nugetVersion.IsPrerelease, FullString = versionFullString, Type = types, LatestCommit = commit }); dbContext.Commits.Add(commit); foreach (var group in dbContext.PackageDependencyGroups.Include(g => g.PackageVersion) .Where(x => x.PackageId == pkgId && x.PackageVersionFullString == versionFullString) .ToList()) { dbContext.PackageDependencyGroups.Remove(group); } version.DependencyGroups = new List(); foreach (var framework in frameWorks) { var group = new PackageDependencyGroup { TargetFramework = framework.FrameworkName, PackageId = pkgId, PackageVersionFullString = versionFullString, Dependencies = framework.Dependencies.Select( d => new Dependency { PackageId = d.PackageId, Version = d.PackageVersion, }).ToList() }; version.DependencyGroups.Add(group); dbContext.PackageDependencyGroups.Add(group); } await dbContext.SaveChangesAsync(); await UpdateCatalogForAsync(commit); using (var shaCrypto = System.Security.Cryptography.SHA512.Create()) { using (var stream = System.IO.File.OpenRead(fullPath)) { var hash = shaCrypto.ComputeHash(stream); var shaFullName = fullPath + ".sha512"; var hashText = Convert.ToBase64String(hash); var hashTextBytes = Encoding.ASCII.GetBytes(hashText); using (var shaFile = System.IO.File.OpenWrite(shaFullName)) { shaFile.Write(hashTextBytes, 0, hashTextBytes.Length); } } } string nugetSpecificationFullPath = Path.Combine(pkgPath, pkgId + "." + Constants.SpecFileExtension); FileInfo nugetSpecificationFullPathInfo = new(nugetSpecificationFullPath); if (nugetSpecificationFullPathInfo.Exists) nugetSpecificationFullPathInfo.Delete(); spec.ExtractToFile(nugetSpecificationFullPath); } } return version; } private FrameworkDependencyGroup NewFrameworkDependencyGroup(XElement x) { var view = x.ToJson(); var frameworkReferences = x.Descendants(); var framework = x.Attribute("targetFramework").Value; var deps = frameworkReferences.Select(r => new ShortDependencyInfo { PackageId = (r.Attribute("name") ?? r.Attribute("id")).Value, PackageVersion = r.Attribute("version")?.Value }); return new FrameworkDependencyGroup { FrameworkName = framework, Dependencies = deps.ToList() }; } } }