isn/src/isnd/Services/PackageManager.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;
using isn.abst;
using isn.Abstract;
using isnd.Data;
using isnd.Data.Catalog;
using isnd.Data.Packages;
using isnd.Entities;
using isnd.Helpers;
using isnd.Interfaces;
using isnd.ViewModels;
using NuGet.Packaging.Core;
using NuGet.Versioning;
using System.Xml;
using System.Xml.Linq;
using System.Threading;
using NuGet.Protocol;

namespace isnd.Services
{
    public class PackageManager : IPackageManager
    {
        public const string BASE_API_LEVEL = "3.0.0";
        readonly ApplicationDbContext dbContext;

        public PackageManager(ApplicationDbContext dbContext,
        IOptions<IsndSettings> siteConfigOptionsOptions)
        {
            this.dbContext = dbContext;
            isndSettings = siteConfigOptionsOptions.Value;
            apiBase = isndSettings.ExternalUrl + Constants.ApiVersionPrefix;
        }

        public IEnumerable<Resource> GetResources()
        {

            var res = new List<Resource>
            {
                new Resource(apiBase + ApiConfig.Package,
                "PackagePublish/2.0.0")
                {
                    Comment = "Package Publish service"
                },
                // under dev, only leash in release mode

                new Resource(apiBase + ApiConfig.Package + "/{id}/{version}",
                "PackageDetailsUriTemplate/5.1.0")
                {
                    Comment = "URI template used by NuGet Client to construct details URL for packages"
                },

                new Resource(apiBase + ApiConfig.Content,
                "PackageBaseAddress/3.0.0")
                {
                    Comment = "Package Base Address service - " +
                     "Base URL of where NuGet packages are stored, in the format " 
                     // "https://<host>/nupkg/{id-lower}/{version-lower}/{id-lower}.{version-lower}.nupkg"
                },

                new Resource(apiBase + ApiConfig.AutoComplete,
                "SearchAutocompleteService/" + BASE_API_LEVEL)
                {
                    Comment = "Auto complete service"
                },

                new Resource(apiBase + ApiConfig.Search,
                "SearchQueryService")
                {
                    Comment = "Query endpoint of NuGet Search service (primary) used by RC clients"
                },
                new Resource(apiBase + ApiConfig.Search,
                "SearchQueryService/3.0.0-beta")
                {
                    Comment = "Query endpoint of NuGet Search service (primary) used by RC clients"
                },
                new Resource(apiBase + ApiConfig.Search,
                "SearchQueryService/3.0.0-rc")
                {
                    Comment = "Query endpoint of NuGet Search service (primary) used by RC clients"
                },
                new Resource(apiBase + ApiConfig.Search,
                "SearchQueryService/3.5.0")
                {
                    Comment = "Query endpoint of NuGet Search service (primary) used by RC clients"
                },

                new Resource(apiBase +  ApiConfig.Registration,
                "RegistrationsBaseUrl/Versioned")
                {
                    Comment = "Base URL of storage where package registration info is stored. " +
                     "This base URL includes SemVer 2.0.0 packages."
                }
            };
            return res;
        }

        public AutoCompleteResult AutoComplete(string id,
             int skip, int take, bool prerelease = false,
             string packageType = null)
        {
            var scope = dbContext.PackageVersions.Where(
                         v => v.PackageId.StartsWith(id)
                            && (prerelease || !v.IsPrerelease)
                            && (packageType == null || v.Type == packageType)
                     )
                     .OrderBy(v => v.FullString);
            return new AutoCompleteResult
            {
                totalHits = scope.Count(),
                data = scope.Select(v => v.FullString)
                     .Skip(skip).Take(take).ToArray()
            };
        }

        public string[] GetVersions(
            string id,
            NuGetVersion parsedVersion = null,
            bool prerelease = false,
            string packageType = null,
            int skip = 0,
            int take = 25)
        {
            return dbContext.PackageVersions.Where(
                         v => v.PackageId == id
                            && (prerelease || !v.IsPrerelease)
                            && (packageType == null || v.Type == packageType)
                            && (parsedVersion == null || parsedVersion.CompareTo
                            (new SemanticVersion(v.Major, v.Minor, v.Patch)) < 0)
                     )
                     .OrderBy(v => v.NugetVersion)
                     .Select(v => v.FullString)
                     .Skip(skip).Take(take).ToArray();
        }

        public string CatalogBaseUrl => apiBase;

        private IsndSettings isndSettings;
        private readonly string apiBase;

        public virtual async Task<PackageRegistration> GetCatalogIndexAsync()
        {
            return await UpdateCatalogForAsync(null);
        }

        public async Task<PackageRegistration> UpdateCatalogForAsync
        (Commit reason = null)
        {
            int i = 0;

            string baseId = apiBase + ApiConfig.Catalog;
            string baseRegistrationId = $"{apiBase}{ApiConfig.Registration}";
            PackageRegistration index = new PackageRegistration(baseId);

            var scope = await dbContext.Commits.OrderBy(c => c.TimeStamp).ToArrayAsync();


            i = isndSettings.CatalogPageLen;
            foreach (var commit in scope)
            {

                var validPkgs = (await dbContext.Packages
                .Include(po => po.Owner)
                .Include(pkg => pkg.Versions)
                .Include(pkg => pkg.LatestCommit)
                .ToArrayAsync())
                .GroupBy((q) => q.Id);

                foreach (var pkgIdGroup in validPkgs)
                {

                    CatalogPage page = index.Items.FirstOrDefault
                    (p => p.GetPackageId() == pkgIdGroup.Key);
                    if (page == null)
                    {
                        page = new CatalogPage(pkgIdGroup.Key, apiBase);
                        index.Items.Add(page);
                    }

                    foreach (var package in pkgIdGroup)
                    {
                        page.AddVersionRange(package.Versions);
                    }
                }
                reason = commit;
                i++;
            }
            return index;
        }

        public async Task<PackageDeletionReport> DeletePackageAsync
        (string pkgId, string version, string type)
        {
            // TODO deletion on disk
            var commit = new Commit
            {
                Action = PackageAction.DeletePackage,
                TimeStamp = DateTimeOffset.Now.ToUniversalTime()
            };
            dbContext.Commits.Add(commit);
            var pkg = await dbContext.PackageVersions.SingleOrDefaultAsync(
                v => v.PackageId == pkgId &&
                v.FullString == version &&
                v.Type == type
            );
            if (pkg == null)
            {
                return new PackageDeletionReport { Deleted = false };
            }
            dbContext.PackageVersions.Remove(pkg);
            await dbContext.SaveChangesAsync();
            await UpdateCatalogForAsync(commit);
            return new PackageDeletionReport { Deleted = true, DeletedVersion = pkg };
        }

        public async Task<PackageVersion> GetPackageAsync
        (string pkgId, string version, string type)
        {
            return await dbContext.PackageVersions.SingleOrDefaultAsync(
                v => v.PackageId == pkgId &&
                v.FullString == version &&
                v.Type == type
            );
        }

        public async Task<Data.Catalog.RegistrationLeave> GetCatalogEntryAsync
        (string pkgId, string semver, string pkgType = null)
        {
            var version = await dbContext.PackageVersions
            .Include(v => v.Package)
            .Include(v => v.Package.LatestCommit)
            .Include(v => v.Package.Owner)
            .Include(v => v.DependencyGroups)
            .Include(v => v.LatestCommit)
              .Where(v => v.PackageId == pkgId
              && v.FullString == semver
              && v.LatestCommit != null
              ).SingleOrDefaultAsync();

            foreach (var g in version.DependencyGroups)
            {
                g.Dependencies = dbContext.Dependencies.Where(d => d.DependencyGroupId == g.Id).ToList();
            }

            return version.ToPackage(apiBase);
        }

        public async Task<PackageDeletionReport> UserAskForPackageDeletionAsync
        (string uid, string id, string lower, string type)
        {
            PackageVersion packageVersion = await dbContext.PackageVersions
             .Include(pv => pv.Package)
             .FirstOrDefaultAsync(m => m.PackageId == id
             && m.FullString == lower && m.Type == type);
            if (packageVersion == null) return null;
            if (packageVersion.Package.OwnerId != uid) return null;
            return new PackageDeletionReport { Deleted = true, DeletedVersion = packageVersion };
        }


        public IEnumerable<Data.Catalog.RegistrationLeave> SearchCatalogEntriesById
        (string pkgId, string semver, string pkgType, bool preRelease)
        {
            // PackageDependency
            return dbContext.PackageVersions
            .Include(v => v.Package)
            .Include(v => v.Package.Owner)
            .Include(v => v.Package.LatestCommit)
            .Include(v => v.LatestCommit)
            .Include(v => v.DependencyGroups)
              .Where(v => v.PackageId == pkgId && semver == v.FullString
               && (pkgType == null || pkgType == v.Type)
               && (preRelease || !v.IsPrerelease))
               .OrderByDescending(p => p.CommitNId)
               .Select(p => p.ToPackage(apiBase))
               ;
        }
        public PackageVersion GetPackage(string pkgId, string semver, string pkgType)
        {
            return dbContext.PackageVersions
            .Include(v => v.Package)
            .Include(v => v.LatestCommit)
            .Include(v => v.DependencyGroups.Last().Dependencies)
              .Single(v => v.PackageId == pkgId && semver == v.FullString
               && (pkgType == null || pkgType == v.Type));
        }

        public async Task<PackageRegistration> GetPackageRegistrationIndexAsync
        (PackageRegistrationQuery query)
        {
            // RegistrationPageIndexAndQuery
            if (string.IsNullOrWhiteSpace(query.Query)) return null;
            query.Query = query.Query.ToLower();
            var scope = await dbContext.PackageVersions
            .Include(p => p.Package)
            .Include(p => p.Package.Owner)
            .Include(p => p.LatestCommit)
            .Where(p => p.PackageId.ToLower() == query.Query).ToArrayAsync();
            if (scope == null) return null;
            if (scope.Length == 0) return null;
            string bid = $"{apiBase}{ApiConfig.Registration}";

            foreach (var version in scope)
            {
                version.DependencyGroups = dbContext.PackageDependencyGroups.Include(d => d.Dependencies)
                    .Where(d => d.PackageId == version.PackageId && d.PackageVersionFullString == version.FullString)
                    .ToList();
                version.LatestCommit = dbContext.Commits.Single(c => c.Id == version.CommitNId);
            }

            return
                new PackageRegistration(apiBase, query.Query, scope);
        }

        public async Task<PackageSearchResult> SearchPackageAsync(PackageRegistrationQuery query)
        {
            string bid = $"{apiBase}{ApiConfig.Registration}";
            if (string.IsNullOrWhiteSpace(query.Query))
                query.Query = "";

            var packages = await dbContext.Packages
            .Include(g => g.Versions)
            .Where(d => d.Id.StartsWith(query.Query)
            && (query.Prerelease || d.Versions.Any(v => !v.IsPrerelease)))
            .Skip(query.Skip).Take(query.Take).ToArrayAsync();
            foreach (var package in packages)
                foreach (var version in package.Versions)
                {
                    version.DependencyGroups = dbContext.PackageDependencyGroups.Include(d => d.Dependencies)
                    .Where(d => d.PackageVersionFullString == version.FullString && d.PackageId == version.PackageId)
                    .ToList();
                }
            return new PackageSearchResult(packages, apiBase, packages.Count());

        }

        public async Task<PackageVersion> PutPackageAsync(Stream packageStream, string ownerId)
        {
            PackageVersion version = null;
            using (packageStream)
            {
                using (var archive = new ZipArchive(packageStream))
                {
                    var spec = archive.Entries.FirstOrDefault(e => e.FullName.EndsWith("." + Constants.SpecFileExtension));
                    if (spec == null) throw new InvalidPackageException("no " + Constants.SpecFileExtension + " from archive");
                    string pkgPath;
                    NuGetVersion nugetVersion;
                    string pkgId;
                    string fullPath;

                    using var specificationStream = spec.Open();

                    using XmlReader xmlReader = XmlReader.Create(specificationStream);

                    var xMeta = XElement.Load(xmlReader, LoadOptions.None).Descendants().First();

                    string packageDescription = xMeta.Descendants().FirstOrDefault(x => x.Name.LocalName == "description")?.Value;

                    var dependencies = xMeta
                    .Descendants().FirstOrDefault(x => x.Name.LocalName == "dependencies");
                    var  frameworkReferences= xMeta
                    .Descendants().FirstOrDefault(x => x.Name.LocalName == "frameworkReferences");

                    var frameWorks = (dependencies ?? frameworkReferences)
                    .Descendants().Where(x => x.Name.LocalName == "group")
                    .Select(x => NewFrameworkDependencyGroup(x)).ToArray();
                    var types = "Package";

                    pkgId = xMeta.Descendants().FirstOrDefault(x => x.Name.LocalName == "id")?.Value;
                    string pkgVersion = xMeta.Descendants().FirstOrDefault(x => x.Name.LocalName == "version")?.Value;

                    if (!NuGetVersion.TryParse(pkgVersion, out nugetVersion))
                        throw new InvalidPackageException("metadata/version");




                    string packageIdPath = Path.Combine(isndSettings.PackagesRootDir,
                        pkgId);
                    pkgPath = Path.Combine(packageIdPath, nugetVersion.ToFullString());
                    string name = $"{pkgId}-{nugetVersion}." + Constants.PacketFileExtension;
                    fullPath = Path.Combine(pkgPath, name);

                    var packageIdPathInfo = new DirectoryInfo(packageIdPath);
                    Data.Packages.Package pkg = dbContext.Packages.SingleOrDefault(p => p.Id == pkgId);
                    Commit commit = new Commit
                    {
                        Action = PackageAction.PublishPackage,
                        TimeStamp = DateTimeOffset.Now.ToUniversalTime()
                    };
                    if (pkg != null)
                    {
                        // Update
                        pkg.Description = packageDescription;
                        pkg.LatestCommit = commit;
                    }
                    else
                    {
                        // First version
                        pkg = new Data.Packages.Package
                        {
                            Id = pkgId,
                            Description = packageDescription,
                            OwnerId = ownerId,
                            LatestCommit = commit
                        };
                        dbContext.Packages.Add(pkg);
                    }

                    // here, the package is or new, or owned by the key owner
                    if (!packageIdPathInfo.Exists) packageIdPathInfo.Create();

                    var dest = new FileInfo(fullPath);
                    var destDir = new DirectoryInfo(dest.DirectoryName);
                    if (dest.Exists) dest.Delete();
                    if (!destDir.Exists) destDir.Create();

                    packageStream.Seek(0, SeekOrigin.Begin);
                    using (var fileStream = File.Create(fullPath))
                    {
                        await packageStream.CopyToAsync(fileStream);
                    }

                    string fullStringVersion = nugetVersion.ToFullString();
                    var pkgVersions = dbContext.PackageVersions.Where
                    (v => v.PackageId == pkg.Id && v.FullString == fullStringVersion);
                    if (pkgVersions.Count() > 0)
                    {
                        foreach (var v in pkgVersions.ToArray())
                            dbContext.PackageVersions.Remove(v);
                    }
                    string versionFullString = nugetVersion.ToFullString();

                    // FIXME default package type or null
                    dbContext.PackageVersions.Add
                    (version = new PackageVersion
                    {
                        Package = pkg,
                        Major = nugetVersion.Major,
                        Minor = nugetVersion.Minor,
                        Patch = nugetVersion.Patch,
                        Revision = nugetVersion.Revision,
                        IsPrerelease = nugetVersion.IsPrerelease,
                        FullString = versionFullString,
                        Type = types,
                        LatestCommit = commit
                    });

                    dbContext.Commits.Add(commit);

                    foreach (var group in dbContext.PackageDependencyGroups.Include(g => g.PackageVersion)
                    .Where(x => x.PackageId == pkgId && x.PackageVersionFullString == versionFullString)
                    .ToList())
                    {
                        dbContext.PackageDependencyGroups.Remove(group);
                    }
                    version.DependencyGroups = new List<PackageDependencyGroup>();
                    foreach (var framework in frameWorks)
                    {
                        var group = new PackageDependencyGroup
                        {
                            TargetFramework = framework.FrameworkName,
                            PackageId = pkgId,
                            PackageVersionFullString = versionFullString,
                            Dependencies = framework.Dependencies.Select(
                                d => new Dependency
                                {
                                    PackageId = d.PackageId,
                                    Version = d.PackageVersion,
                                }).ToList()
                        };

                        version.DependencyGroups.Add(group);
                        dbContext.PackageDependencyGroups.Add(group);

                    }
                    await dbContext.SaveChangesAsync();
                    await UpdateCatalogForAsync(commit);


                    using (var shaCrypto = System.Security.Cryptography.SHA512.Create())
                    {
                        using (var stream = System.IO.File.OpenRead(fullPath))
                        {
                            var hash = shaCrypto.ComputeHash(stream);
                            var shaFullName = fullPath + ".sha512";
                            var hashText = Convert.ToBase64String(hash);
                            var hashTextBytes = Encoding.ASCII.GetBytes(hashText);

                            using (var shaFile = System.IO.File.OpenWrite(shaFullName))
                            {
                                shaFile.Write(hashTextBytes, 0, hashTextBytes.Length);
                            }
                        }
                    }
                    string nugetSpecificationFullPath = Path.Combine(pkgPath, pkgId + "." + Constants.SpecFileExtension);
                    FileInfo nugetSpecificationFullPathInfo = new(nugetSpecificationFullPath);

                    if (nugetSpecificationFullPathInfo.Exists)
                        nugetSpecificationFullPathInfo.Delete();
                    spec.ExtractToFile(nugetSpecificationFullPath);
                }
            }
            return version;
        }

        private FrameworkDependencyGroup NewFrameworkDependencyGroup(XElement x)
        {
            var view = x.ToJson();
            var frameworkReferences = x.Descendants();
            var framework = x.Attribute("targetFramework").Value;
            var deps = frameworkReferences.Select(r => new ShortDependencyInfo
                {
                    PackageId = (r.Attribute("name") ?? r.Attribute("id")).Value,
                    PackageVersion = r.Attribute("version")?.Value
                });

            return new FrameworkDependencyGroup
            {
                FrameworkName = framework,
                Dependencies = deps.ToList()
            };

        }
    }
}