jetbrains: include small set of generated IDEA plugins

Some edge cases but appears to work on my machine (TM).

Still TODO is setting up the job to generate and commit updates.

bin/update-jetbrains-plugins.py

@@ -27,6 +27,10 @@ PRODUCT_CODE = {
 }
 
 
+PACKAGE_RE = re.compile("[^0-9A-Za-z._-]")
+HTML_RE = re.compile("<[^>]+/?>")
+
+
 def to_slug(name):
     slug = name.replace(" ", "-").lstrip(".")
     for char in ",/;'\\<>:\"|!@#$%^&*()":
@@ -59,18 +63,15 @@ class Build:
         return self.code + "-" + self.version
 
 
-PACKAGE_RE = re.compile("[^0-9A-Za-z._-]")
-HTML_RE = re.compile("<[^>]+/?>")
-
-
 class Plugin:
-    def __init__(self, data, category=None):
+    def __init__(self, data, build, category=None):
+        self.build = build
         self.category = category
         self.name = data.find("name").text
-        self.id = data.find("id").text
+        self.xml_id = data.find("id").text
         self._description = data.find("description").text
         self.url = data.get("url") or data.find("vendor").get("url")
-        self.version = data.find("version").text
+        self.version = data.find("version").text.replace(" ", "-")
         self.slug = to_slug(self.name)
         self.orig_slug = self.slug
 
@@ -84,7 +85,7 @@ class Plugin:
     def description(self):
         return re.sub(HTML_RE, "", self._description or "").strip()
 
-    def download_url(self, build, deref=True):
+    def get_download_url(self, deref=True):
         """
         Provides the ZIP download URL for this plugin.
 
@@ -94,18 +95,52 @@ class Plugin:
         (which it is by default). However, this comes at the cost of requiring
         an HTTP request.
         """
-        id = urllib.parse.quote(self.id)
-        url = f"https://plugins.jetbrains.com/pluginManager?action=download&id={id}&build={build}"
+        id = urllib.parse.quote(self.xml_id)
+        url = f"https://plugins.jetbrains.com/pluginManager?action=download&id={id}&build={self.build}"
         if deref:
             res = requests.get(url, allow_redirects=not deref)
             url = "https://plugins.jetbrains.com" + re.sub(
                 "\?.*$", "", res.headers["location"]
             )
+            self.jetbrains_url = url
             if url.endswith("external"):
                 res = requests.get(url, allow_redirects=not deref)
                 url = res.headers["location"]
         return url
 
+    def fetch_external(self, update_only=False):
+        """
+        Performs network calls to update this plugin with information that
+        cannot be performed from the public XML API.
+
+        Additional attributes provided after this method:
+
+          download_url : the plugin download location
+          sha          : the SHA256 of the download source
+
+        If update_only is true, a full update is performed, also providing:
+
+          id          : the plugin integer ID
+          license_url : the plugin license URL
+          license     : the Nixpkgs license attribute
+        """
+        self.download_url = self.get_download_url(deref=True)
+        self.sha = prefetch(self, self.build, self.download_url)
+
+        if update_only:
+            return
+
+        self.id = self.jetbrains_url.split("/")[4]
+        res = requests.get(
+            f"https://plugins.jetbrains.com/api/plugins/{self.id}"
+        ).json()
+        try:
+            self.url = self.url or res["urls"]["sourceCodeUrl"]
+        except KeyError:
+            pass
+        self.license_url = res["urls"]["licenseUrl"]
+        self.license = translate_license(self.license_url, fallback=self.url)
+
     def packagename(self):
         slug = re.sub(PACKAGE_RE, "", self.slug.lower()).replace(".", "-")
         if slug[0] in "1234567890":
@@ -128,16 +163,16 @@ def list_plugins(build):
     https://plugins.jetbrains.com/docs/marketplace/plugins-list.html
     """
     resp = requests.get(f"https://plugins.jetbrains.com/plugins/list/?build={build}")
-    return parse_repository(resp.content)
+    return parse_repository(resp.content, build)
 
 
-def parse_repository(content):
+def parse_repository(content, build):
     tree = etree.XML(content)
     plugins = []
     for cat in tree.findall("category"):
         cat_name = cat.get("name")
         for plugin in cat.findall("idea-plugin"):
-            plugins.append(Plugin(plugin, cat_name))
+            plugins.append(Plugin(plugin, build, cat_name))
     return plugins
 
 
@@ -155,7 +190,7 @@ def deduplicate(plugins):
 
 def prefetch(plugin, build, url=None):
     if not url:
-        url = plugin.download_url(build)
+        url = plugin.download_url or plugin.get_download_url()
     res = sp.run(
         ["nix-prefetch-url", "--name", plugin.filename(), url], capture_output=True,
     )
@@ -164,19 +199,90 @@ def prefetch(plugin, build, url=None):
     return res.stdout.decode("utf-8").strip()
 
 
-def write_packages(outfile, plugins, build):
-    builder = build.builder()
+def custom_license(short, full, url, free=False):
+    return f"""{{
+        shortName = "{short}";
+        fullName = "{full}";
+        url = "{url}";
+        free = {"true" if free else "false"};
+      }}"""
+
+
+def arr(url):
+    return custom_license("allrightsreserved", "All Rights Reserved", url)
+
+
+def translate_license(url, fallback=""):
+    license = url.lower()
+    if license == "":
+        print(f"no license for {fallback}", file=sys.stderr)
+        return arr(fallback)
+    # Common (license) hosts
+    elif "github.com" in license or "raw.githubusercontent.com" in license:
+        try:
+            owner, repo = url.split("/")[3:5]
+        except ValueError:
+            print(f"no license metadata for {url}", file=sys.stderr)
+            return arr(url)
+        res = requests.get(
+            f"https://api.github.com/repos/{owner}/{repo}",
+            headers={"Accept": "application/vnd.github.v3+json"},
+        ).json()
+        try:
+            return translate_license(res["license"]["key"])
+        except (KeyError, TypeError):
+            print(f"no license metadata for {url}", file=sys.stderr)
+            return arr(url)
+    elif "opensource.org" in license:
+        os_license = license.rstrip("/").split("/")[-1]
+        if os_license == "alphabetical":
+            # Doesn't actually have a license, it's the listing page
+            return arr(fallback)
+        return translate_license(os_license)
+    # Actual translations now
+    elif "apache.org/licenses/license-2.0" in license or "apache-2.0" in license:
+        return "lib.licenses.asl20"
+    elif "artistic-2" in license:
+        return "lib.licenses.artistic2"
+    elif "bsd-2-clause" in license:
+        return "lib.licenses.bsd2"
+    elif "bsd-3-clause" in license:
+        return "lib.licenses.bsd3"
+    elif "eclipse.org/legal/epl-2.0" in license:
+        return "lib.licenses.epl20"
+    elif "gpl-3.0" in license:
+        return "lib.licenses.gpl3Only"
+    elif "mit" in license:
+        return "lib.licenses.mit"
+    elif "osd" in license:
+        return "lib.licenses.free"
+    elif "other" == license:
+        return arr(fallback)
+    # Custom known licenses
+    elif "plugins.jetbrains.com/legal/terms-of-use" in license:
+        return custom_license(
+            "jetbrains", "Jetbrains Plugin Marketplace Agreement", license
+        )
+    # Fallback
+    else:
+        print(f"unrecognised license {license}", file=sys.stderr)
+        return arr(license)
+
+
+def write_packages(outfile, plugins):
+    builder = plugins[0].build.builder() or ""
     outfile.write("{callPackage}:\n{")
 
-    for plugin in plugins:
-        src_url = plugin.download_url(build, deref=True)
-        src_ext = os.path.splitext(src_url)[-1]
-
+    for i, plugin in enumerate(plugins):
+        print(f"{i:04} {plugin.packagename()}")
         try:
-            sha = prefetch(plugin, build, src_url)
+            plugin.fetch_external()
         except IOError as e:
             print(e, file=sys.stderr)
             continue
+        src_url = plugin.download_url
+        src_ext = os.path.splitext(src_url)[-1]
+        sha = plugin.sha
 
         build_inputs = []
         if src_ext == ".zip":
@@ -186,7 +292,7 @@ def write_packages(outfile, plugins, build):
         # internal and external plugins; need to find some way to resolve them
         requires = []
         # TODO: Licenses are actually on the website, but aren't provided in the API
-        license = "lib.licenses.free"
+        license = plugin.license
 
         call_args = [str(builder), "fetchurl", "lib"]
         for binput in build_inputs:
@@ -197,7 +303,7 @@ def write_packages(outfile, plugins, build):
   {plugin.packagename()} = callPackage ({{ {", ".join(sorted(call_args))} }}: {builder} {{
     pname = "{plugin.slug}";
     plugname = "{plugin.name}";
-    plugid = "{plugin.id}";
+    plugid = "{plugin.xml_id}";
     version = "{plugin.version}";
     src = fetchurl {{
       url = "{src_url}";
@@ -227,6 +333,7 @@ def main():
     parser.add_argument(
         "-o", "--out", type=str, help="File to write plugins to",
     )
+    parser.add_argument("-O", "--offset", type=int, help="Offset number of packages")
     parser.add_argument(
         "package",
         metavar="PACKAGE",
@@ -241,15 +348,17 @@ def main():
     plugins.sort(key=lambda p: p.slug)
     deduplicate(plugins)
 
+    if args.offset:
+        plugins = plugins[args.offset :]
     if args.number:
         plugins = plugins[: args.number]
 
     print(f"Generating packages for {len(plugins)} plugins", file=sys.stderr)
     if not args.out:
-        write_packages(sys.stdout, plugins, build)
+        write_packages(sys.stdout, plugins)
     else:
         with open(args.out, "w") as f:
-            write_packages(f, plugins, build)
+            write_packages(f, plugins)
 
 
 main()

pkgs/applications/editors/jetbrains/idea-plugins.nix

@@ -10,7 +10,7 @@ let
   };
 
   generateIdea = lib.makeOverridable ({
-    idea ? ./manual-idea-packages.nix
+    idea ? ./idea-generated.nix
   }: let
 
     imported = import idea {

pkgs/applications/editors/jetbrains/manual-common-packages.nix

@@ -1,33 +0,0 @@
-{ callPackage }:
-{
-  ideavim = callPackage
-    ({ commonBuild }: commonBuild {
-      pname = "IdeaVim";
-      version = "0.57";
-      pluginId = 164;
-      versionId = 85009;
-      sha256 = "1rwfwj0b0nwi7jxhzxk1r0xc190nf4i3b59i0zknpmgb4yc5clzw";
-    })
-    { };
-
-  checkstyle-idea = callPackage
-    ({ commonBuild }: commonBuild {
-      pname = "CheckStyle-IDEA";
-      version = "5.42.0";
-      pluginId = 1065;
-      versionId = 95757;
-      sha256 = "0sji3649n5zz84dlidqaklipq6vaiafxsvg0gzy3j59mvkz6dk14";
-    })
-    { };
-
-  google-java-format = callPackage
-    ({ commonBuild }: commonBuild rec {
-      pname = "google-java-format";
-      version = "1.7.0.4";
-      pluginId = 8527;
-      versionId = 83164;
-      sha256 = "1pmnn1ksiv44kdga53gi3psrm2sva4bqrxizagbr0if2n0rrvgii";
-      filename = "${pname}.zip";
-    })
-    { };
-}

pkgs/applications/editors/jetbrains/manual-idea-packages.nix

@@ -1,12 +0,0 @@
-{ callPackage }:
-{
-  spring-assistant = callPackage
-    ({ ideaBuild }: ideaBuild {
-      pname = "intellij-spring-assistant";
-      version = "0.12.0";
-      pluginId = 10229;
-      versionId = 44968;
-      sha256 = "13cglywzhb4j0qj0bs2jwaz2k8pxrxalv35wgkmgkxr635bxmwsj";
-    })
-    { };
-}

pkgs/applications/editors/jetbrains/wrapper.nix

@@ -1,5 +1,4 @@
-{ lib
-}: self:
+{ lib, makeWrapper, runCommand }: self:
 
 with lib;
 
@@ -20,12 +19,31 @@ in
 
 assert assertMsg (length badPlugins == 0) errorMsg;
 
-appendToName "with-plugins" (package.overrideAttrs (oldAttrs: {
-  passthru = { inherit plugins; };
-  # TODO: Purely aesthetics, but link the plugin to its name instead of hash-name-version
-  installPhase = oldAttrs.installPhase + ''
-    for plugin in $plugins; do
-      ln -s "$plugin" "$out/$name/plugins/$(basename $plugin)"
-    done
-  '';
-}))
+runCommand
+  (appendToName "with-plugins" package).name
+{
+  nativeBuildInputs = [ package makeWrapper ];
+  inherit package plugins;
+  packageName = package.name;
+
+  preferLocalBuild = true;
+  allowSubstitutes = false;
+
+} ''
+  mkdir -p $out/$packageName/plugins
+  for dir in $package/*; do
+    cp -r $dir $out/
+  done
+
+  # Install plugins
+  for plugin in $plugins; do
+    local pluginName=$(basename $plugin)
+    pluginName=''${pluginName#*-}
+    pluginName=''${pluginName%-([0-9].)*}
+    ln -s $plugin $out/$packageName/plugins/$pluginName
+  done
+
+  # Fix up wrapper
+  substituteInPlace $out/bin/* \
+    --replace "$package" "$out"
+''

pkgs/build-support/jetbrains/plugin.nix

@@ -4,30 +4,38 @@
 , jetbrainsPlatforms
 }:
 
-{ pluginId
+{ plugid
 , pname
 , version
-, versionId
-, sha256
-, filename ? "${pname}-${version}.zip"
-}:
+, ...
+}@args:
 
-stdenv.mkDerivation {
-  inherit pname version;
+let
 
-  src = fetchzip {
-    inherit sha256;
-    url = "https://plugins.jetbrains.com/files/${toString pluginId}/${toString versionId}/${filename}";
+  defaultMeta = {
+    broken = false;
+  } // lib.optionalAttrs ((args.src.meta.homepage or "") != "") {
+    homepage = args.src.meta.homepage;
+  } // lib.optionalAttrs ((args.src.meta.description or "") != "") {
+    description = args.src.meta.description;
+  } // lib.optionalAttrs ((args.src.meta.license or {}) != {}) {
+    license = args.src.meta.license;
   };
 
+in
+
+stdenv.mkDerivation (args // {
   passthru = { inherit jetbrainsPlatforms; };
 
+  dontUnpack = lib.any (lib.hasSuffix ".jar") args.src.urls;
+
   installPhase = ''
     mkdir $out
     cp -r * $out/
   '';
 
   meta = {
-    homepage = "https://plugins.jetbrains.com/plugin/${pluginId}-${lib.toLower pname}";
+    inherit (args.meta) license description;
+    homepage = if (args.meta.homepage == "") then null else args.meta.homepage;
   };
-}
+})

pkgs/top-level/all-packages.nix

@@ -55,7 +55,7 @@ rec {
   # A functional Jetbrains IDE-with-plugins package set.
   jetbrains = pkgs.dontRecurseIntoAttrs rec {
     jetbrainsPluginsFor = variant: import ../top-level/jetbrains-plugins.nix {
-      inherit (pkgs) lib newScope stdenv fetchzip;
+      inherit (pkgs) lib newScope stdenv fetchzip makeWrapper runCommand;
       inherit variant;
     };
 

pkgs/top-level/jetbrains-plugins.nix

@@ -2,29 +2,25 @@
 , newScope
 , stdenv
 , fetchzip
+, makeWrapper
+, runCommand
 
 , variant
 }:
 
 let
 
-  mkJetbrainsPlugins = import ../applications/editors/jetbrains/common-plugins.nix {
-    inherit lib stdenv fetchzip;
-  };
-
   mkIdeaPlugins = import ../applications/editors/jetbrains/idea-plugins.nix {
     inherit lib stdenv fetchzip;
   };
 
   jetbrainsWithPlugins = import ../applications/editors/jetbrains/wrapper.nix {
-    inherit lib;
+    inherit lib makeWrapper runCommand;
   };
 
 in lib.makeScope newScope (self: lib.makeOverridable ({
-  jetbrainsPlugins ? mkJetbrainsPlugins self
-  , ideaPlugins ? mkIdeaPlugins self
+  ideaPlugins ? mkIdeaPlugins self
 }: ({ }
-  // jetbrainsPlugins // { inherit jetbrainsPlugins; }
   // ideaPlugins // { inherit ideaPlugins; }
   // {
     inherit variant;