Allow more complex repo-level metadata additions

Previously we only allowed removing components, now the repository
vendor can also inject arbitrary additional data into <custom/> tags.

The only current limitation is that data needs to be reprocessed for a
particular component so changes to a <custom/> tag are applied.

src/asgen/cptmodifiers.d

@@ -0,0 +1,167 @@
+/*
+ * Copyright (C) 2021-2022 Matthias Klumpp <matthias@tenstral.net>
+ *
+ * Licensed under the GNU Lesser General Public License Version 3
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the license, or
+ * (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this software.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+module asgen.cptmodifiers;
+
+import core.sync.rwmutex : ReadWriteMutex;
+import std.json : parseJSON;
+import std.stdio : File;
+import std.path : buildPath;
+import std.typecons : Nullable;
+static import std.file;
+
+import appstream.Component : Component, ComponentKind, MergeKind;
+
+import asgen.logging;
+import asgen.config : Suite;
+import asgen.result : GeneratorResult;
+
+/**
+ * Helper class to provide information about repository-specific metadata modifications.
+ * Instances of this class must be thread safe.
+ */
+class InjectedModifications
+{
+private:
+    Component[string] m_removedComponents;
+    string[string][string] m_injectedCustomData;
+
+    bool m_hasRemovedCpts;
+    bool m_hasInjectedCustom;
+
+    ReadWriteMutex m_mutex;
+
+public:
+    this ()
+    {
+        m_mutex = new ReadWriteMutex;
+    }
+
+    void loadForSuite (Suite suite)
+    {
+        synchronized (m_mutex.writer) {
+            m_removedComponents.clear ();
+            m_injectedCustomData.clear ();
+
+            immutable fname = buildPath (suite.extraMetainfoDir, "modifications.json");
+            if (!std.file.exists (fname))
+                return;
+            logInfo ("Using repo-level modifications for %s (via modifications.json)", suite.name);
+
+            auto f = File (fname, "r");
+            string jsonData;
+            string line;
+            while ((line = f.readln ()) !is null)
+                jsonData ~= line;
+
+            auto jroot = parseJSON (jsonData);
+
+            if ("InjectCustom" in jroot) {
+                logDebug ("Using injected custom entries from %s", fname);
+                auto jInjCustom = jroot["InjectCustom"].object;
+                foreach (ref jEntry; jInjCustom.byKeyValue) {
+                    string[string] kv;
+                    foreach (ref jCustom; jEntry.value.object.byKeyValue)
+                        kv[jCustom.key] = jCustom.value.str;
+                    m_injectedCustomData[jEntry.key] = kv;
+                }
+            }
+
+            if ("Remove" in jroot) {
+                logDebug ("Using package removal info from %s", fname);
+                foreach (jCid; jroot["Remove"].array) {
+                    immutable cid = jCid.str;
+
+                    auto cpt = new Component;
+                    cpt.setKind (ComponentKind.GENERIC);
+                    cpt.setMergeKind (MergeKind.REMOVE_COMPONENT);
+                    cpt.setId (cid);
+
+                    m_removedComponents[cid] = cpt;
+                }
+            }
+
+            m_hasRemovedCpts = m_removedComponents.length != 0;
+            m_hasInjectedCustom = m_injectedCustomData.length != 0;
+        }
+    }
+
+    @property
+    bool hasRemovedComponents ()
+    {
+        return m_hasRemovedCpts;
+    }
+
+    /**
+     * Test if component was marked for deletion.
+     */
+    bool isComponentRemoved (const string cid)
+    {
+        if (!m_hasRemovedCpts)
+            return false;
+        synchronized (m_mutex.reader)
+            return (cid in m_removedComponents) !is null;
+    }
+
+    /**
+     * Get injected custom data entries.
+     */
+    Nullable!(string[string]) injectedCustomData (const string cid)
+    {
+        Nullable!(string[string]) result;
+        if (!m_hasInjectedCustom)
+            return result;
+        synchronized (m_mutex.reader) {
+            auto injCustomP = cid in m_injectedCustomData;
+            if (injCustomP is null)
+                return result;
+            result = *injCustomP;
+            return result;
+        }
+    }
+
+    void addRemovalRequestsToResult (GeneratorResult gres)
+    {
+        synchronized (m_mutex.reader) {
+            foreach (cpt; m_removedComponents.byValue)
+                gres.addComponentWithString (cpt, gres.pkid ~ "/-" ~ cpt.getId);
+        }
+    }
+
+}
+
+unittest
+{
+    import std.stdio : writeln;
+    import asgen.utils : getTestSamplesDir;
+    writeln ("TEST: ", "InjectedModifications");
+
+    Suite dummySuite;
+    dummySuite.name = "dummy";
+    dummySuite.extraMetainfoDir = buildPath (getTestSamplesDir (), "extra-metainfo");
+
+    auto injMods = new InjectedModifications;
+    injMods.loadForSuite (dummySuite);
+
+    assert (injMods.isComponentRemoved ("com.example.removed"));
+    assert (!injMods.isComponentRemoved ("com.example.not_removed"));
+
+    assert (injMods.injectedCustomData ("org.example.nodata").isNull);
+    assert (injMods.injectedCustomData ("org.example.newdata") == ["earth": "moon", "mars": "phobos", "saturn": "thrym"]);
+}

src/asgen/datastore.d

@@ -72,7 +72,7 @@ public:
     this ()
     {
         opened = false;
-        mdata = new Metadata ();
+        mdata = new Metadata;
         mdata.setLocale ("ALL");
         mdata.setFormatVersion (Config.get ().formatVersion);
         mdata.setWriteHeader(false);

src/asgen/engine.d

@@ -39,6 +39,7 @@ import asgen.result;
 import asgen.hint;
 import asgen.reportgenerator;
 import asgen.localeunit : LocaleUnit;
+import asgen.cptmodifiers : InjectedModifications;
 import asgen.utils : copyDir, stringArrayToByteArray, getCidFromGlobalID;
 
 import asgen.backends.interfaces;
@@ -144,10 +145,11 @@ public:
      * Extract metadata from a software container (usually a distro package).
      * The result is automatically stored in the database.
      */
-    private void processPackages (ref Package[] pkgs, IconHandler iconh)
+    private void processPackages (ref Package[] pkgs, IconHandler iconh, InjectedModifications injMods)
     {
         import std.range : chunks;
         import glib.Thread : Thread;
+
         auto localeUnit = new LocaleUnit (cstore, pkgs);
 
         size_t chunkSize = pkgs.length / Thread.getNumProcessors () / 10;
@@ -158,7 +160,10 @@ public:
         logDebug ("Analyzing %s packages in batches of %s", pkgs.length, chunkSize);
 
         foreach (pkgsChunk; parallel (pkgs.chunks (chunkSize), 1)) {
-            auto mde = new DataExtractor (dstore, iconh, localeUnit);
+            auto mde = new DataExtractor (dstore,
+                                          iconh,
+                                          localeUnit,
+                                          injMods);
 
             foreach (ref pkg; pkgsChunk) {
                 immutable pkid = pkg.id;
@@ -584,53 +589,28 @@ public:
         return pkgMap;
     }
 
-    /**
-     * Read a dedicated JSON file which contains information on which components
-     * to remove from preexisting metadata.
-     */
-    private void readRemovedComponentsInfo (string metainfoDir, ref GeneratorResult gres)
-    {
-        import std.json : parseJSON;
-        import std.stdio : File;
-
-        immutable fname = buildPath (metainfoDir, "removed-components.json");
-        if (!std.file.exists (fname))
-            return;
-
-        auto f = File (fname, "r");
-        string jsonData;
-        string line;
-        while ((line = f.readln ()) !is null)
-            jsonData ~= line;
-
-        auto jroot = parseJSON (jsonData);
-        foreach (jCid; jroot.array) {
-            immutable cid = jCid.str;
-
-            auto cpt = new Component;
-            cpt.setKind (ComponentKind.GENERIC);
-            cpt.setMergeKind (MergeKind.REMOVE_COMPONENT);
-            cpt.setId (cid);
-
-            gres.addComponentWithString (cpt, metainfoDir ~ "/-" ~ cid);
-        }
-    }
-
     /**
      * Read metainfo and auxiliary data injected by the person running the data generator.
      */
-    private Package processExtraMetainfoData (Suite suite, IconHandler iconh, const string section, const string arch)
+    private Package processExtraMetainfoData (Suite suite,
+                                              IconHandler iconh,
+                                              const string section,
+                                              const string arch,
+                                              InjectedModifications injMods)
     {
         import asgen.datainjectpkg : DataInjectPackage;
         import asgen.utils : existsAndIsDir;
 
-        if (suite.extraMetainfoDir is null)
+        if (suite.extraMetainfoDir is null && !injMods.hasRemovedComponents)
             return null;
 
         immutable extraMIDir = buildNormalizedPath (suite.extraMetainfoDir, section);
         immutable archExtraMIDir = buildNormalizedPath (extraMIDir, arch);
 
-        logInfo ("Loading additional metainfo from local directory for %s/%s/%s", suite.name, section, arch);
+        if (suite.extraMetainfoDir is null)
+            logInfo ("Injecting component removal requests for %s/%s/%s", suite.name, section, arch);
+        else
+            logInfo ("Loading additional metainfo from local directory for %s/%s/%s", suite.name, section, arch);
 
         // we create a dummy package to hold information for the injected components
         auto diPkg = new DataInjectPackage (EXTRA_METAINFO_FAKE_PKGNAME, arch);
@@ -644,9 +624,12 @@ public:
         dstore.removePackage (diPkg.id);
 
         // analyze our dummy package just like all other packages
-        auto mde = new DataExtractor (dstore, iconh, null);
+        auto mde = new DataExtractor (dstore, iconh, null, null);
         auto gres = mde.processPackage (diPkg);
 
+        // add removal requests, as we can remove packages from frozen suites via overlays
+        injMods.addRemovalRequestsToResult (gres);
+
         // write resulting data into the database
         dstore.addGeneratorResult (this.conf.metadataType, gres, true);
 
@@ -662,6 +645,15 @@ public:
         if (reportgen is null)
             reportgen = new ReportGenerator (dstore);
 
+        // load repo-level modifications
+        auto injMods = new InjectedModifications;
+        try {
+            injMods.loadForSuite (suite);
+        } catch (Exception e) {
+            throw new Exception (format ("Unable to read modifications.json for suite %s: %s", suite.name, e.msg));
+        }
+
+        // process packages by architecture
         auto sectionPkgs = appender!(Package[]);
         auto suiteDataChanged = false;
         foreach (ref arch; suite.architectures) {
@@ -680,10 +672,10 @@ public:
                                           dstore.mediaExportPoolDir,
                                           getIconCandidatePackages (suite, section, arch),
                                           suite.iconTheme);
-            processPackages (pkgs, iconh);
+            processPackages (pkgs, iconh, injMods);
 
             // read injected data and add it to the database as a fake package
-            auto fakePkg = processExtraMetainfoData (suite, iconh, section, arch);
+            auto fakePkg = processExtraMetainfoData (suite, iconh, section, arch, injMods);
             if (fakePkg !is null)
                 pkgs ~= fakePkg;
 
@@ -702,7 +694,7 @@ public:
             gcCollect ();
         }
 
-
+        // finalize
         if (suiteDataChanged) {
             // export icons for the found packages in this section
             exportIconTarballs (suite, section, sectionPkgs.data);
@@ -817,7 +809,7 @@ public:
                                           getIconCandidatePackages (suite, sectionName, arch),
                                           suite.iconTheme);
             auto pkgsList = pkgs.data;
-            processPackages (pkgsList, iconh);
+            processPackages (pkgsList, iconh, null);
         }
 
         return true;

src/asgen/extractor.d

@@ -44,6 +44,7 @@ import asgen.iconhandler : IconHandler;
 import asgen.utils : componentGetRawIcon, toStaticGBytes;
 import asgen.packageunit : PackageUnit;
 import asgen.localeunit : LocaleUnit;
+import asgen.cptmodifiers : InjectedModifications;
 
 
 final class DataExtractor
@@ -58,10 +59,14 @@ private:
 
     IconHandler iconh;
     LocaleUnit l10nUnit;
+    InjectedModifications modInj;
 
 public:
 
-    this (DataStore db, IconHandler iconHandler, LocaleUnit localeUnit)
+    this (DataStore db,
+         IconHandler iconHandler,
+         LocaleUnit localeUnit,
+         InjectedModifications modInjInfo = null)
     {
         import std.conv : to;
 
@@ -70,6 +75,7 @@ public:
         conf = Config.get ();
         dtype = conf.metadataType;
         l10nUnit = localeUnit;
+        modInj = modInjInfo;
 
         compose = new Compose;
         //compose.setPrefix ("/usr");
@@ -341,6 +347,22 @@ public:
         for (uint i = 0; i < cptsPtrArray.len; i++) {
             auto cpt = new Component (cast (AsComponent*) cptsPtrArray.index (i));
             immutable ckind = cpt.getKind;
+            immutable cid = cpt.getId;
+
+            if (modInj !is null) {
+                // drop component that the repository owner wants to remove
+                if (modInj.isComponentRemoved (cid)) {
+                    gres.removeComponent (cpt);
+                    continue;
+                }
+
+                // inject custom fields from the repository owner, if we have any
+                auto injectedCustom = modInj.injectedCustomData (cid);
+                if (!injectedCustom.isNull) {
+                    foreach (ref ckv; injectedCustom.get.byKeyValue)
+                        cpt.insertCustomValue (ckv.key, ckv.value);
+                }
+            }
 
             if (cpt.getMergeKind != MergeKind.NONE)
                 continue;

src/asgen/meson.build

@@ -18,6 +18,7 @@ asgen_sources = [
     'bindings/lmdb.d',
     'config.d',
     'contentsstore.d',
+    'cptmodifiers.d',
     'datainjectpkg.d',
     'datastore.d',
     'downloader.d',

tests/samples/extra-metainfo/modifications.json

@@ -0,0 +1,9 @@
+{
+  "Remove": [
+    "com.example.removed"
+  ],
+
+  "InjectCustom": {
+    "org.example.newdata": {"earth": "moon", "mars": "phobos", "saturn": "thrym"}
+  }
+}