jetbrains: update IDEA for generated plugins

.github/workflows/update-jetbrains.yml

@@ -1,24 +0,0 @@
-name: "Update Jetbrains plugins"
-on:
-  schedule:
-    - cron: '00 2 * * *'
-
-jobs:
-  tests:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2.3.3
-    - name: Install nix
-      uses: cachix/install-nix-action@v11
-      with:
-        nix_path: "${{ matrix.nixPath }}"
-    - name: Show nixpkgs version
-      run: nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version'
-    - name: Generate package list
-      # TODO switch to default nixpkgs channel once nix-build-uncached 1.0.0 is in stable
-      run: nix run -I 'nixpkgs=channel:nixos-unstable' nixpkgs.nix-build-uncached -c nix-build-uncached ci.nix -A cacheOutputs
-    - name: Trigger NUR update
-      if: ${{ matrix.nurRepo != '<YOUR_REPO_NAME>' }}
-      run: curl -XPOST "https://nur-update.herokuapp.com/update?repo=${{ matrix.nurRepo }}"
-      

bin/update-jetbrains-plugins.py

@@ -2,6 +2,7 @@
 #! nix-shell -i python3 -p python3Packages.lxml python3Packages.requests
 
 import argparse
+import os
 import re
 import requests
 import subprocess as sp
@@ -58,15 +59,20 @@ class Build:
         return self.code + "-" + self.version
 
 
+PACKAGE_RE = re.compile("[^0-9A-Za-z._-]")
+HTML_RE = re.compile("<[^>]+/?>")
+
+
 class Plugin:
     def __init__(self, data, category=None):
         self.category = category
         self.name = data.find("name").text
         self.id = data.find("id").text
-        self.description = data.find("description").text
+        self._description = data.find("description").text
         self.url = data.get("url") or data.find("vendor").get("url")
         self.version = data.find("version").text
         self.slug = to_slug(self.name)
+        self.orig_slug = self.slug
 
         self.depends = []
         for depend in data.findall("depends"):
@@ -75,19 +81,44 @@ class Plugin:
     def __repr__(self):
         return f"<Plugin '{self.name}' {self.version}>"
 
-    def download_url(self, build, follow_redirect=False):
+    def description(self):
+        return re.sub(HTML_RE, "", self._description or "").strip()
+
+    def download_url(self, build, deref=True):
+        """
+        Provides the ZIP download URL for this plugin.
+
+        The trivial URL fetches the latest version through a redirect for some
+        build code and provides no locking to a version for the URL. To fetch
+        a stable URL that can be used as a package source, deref must be set
+        (which it is by default). However, this comes at the cost of requiring
+        an HTTP request.
+        """
         id = urllib.parse.quote(self.id)
-        return f"https://plugins.jetbrains.com/pluginManager?action=download&id={id}&build={build}"
+        url = f"https://plugins.jetbrains.com/pluginManager?action=download&id={id}&build={build}"
+        if deref:
+            res = requests.get(url, allow_redirects=not deref)
+            url = "https://plugins.jetbrains.com" + re.sub(
+                "\?.*$", "", res.headers["location"]
+            )
+            if url.endswith("external"):
+                res = requests.get(url, allow_redirects=not deref)
+                url = res.headers["location"]
+        return url
 
     def packagename(self):
-        slug = self.slug.lower().replace(".", "-")
+        slug = re.sub(PACKAGE_RE, "", self.slug.lower()).replace(".", "-")
         if slug[0] in "1234567890":
             return "_" + slug
         else:
             return slug
 
     def filename(self):
-        return f"{self.slug}-{self.version}.jar"
+        """
+        Returns this plugin's filename without an extension. Rely on the
+        download URL to know the extension.
+        """
+        return f"{self.slug}-{self.version}"
 
 
 def list_plugins(build):
@@ -110,10 +141,23 @@ def parse_repository(content):
     return plugins
 
 
-def prefetch(plugin, build):
+def deduplicate(plugins):
+    """
+    Ensures that the plugin list has unique slugs. Modifies the list in-place.
+    """
+    prev = plugins[0]
+    for plugin in plugins[1:]:
+        if plugin.orig_slug == prev.orig_slug:
+            prev.slug = prev.orig_slug + "-" + prev.version.replace(".", "_")
+            plugin.slug = plugin.orig_slug + "-" + plugin.version.replace(".", "_")
+        prev = plugin
+
+
+def prefetch(plugin, build, url=None):
+    if not url:
+        url = plugin.download_url(build)
     res = sp.run(
-        ["nix-prefetch-url", "--name", plugin.filename(), plugin.download_url(build)],
+        ["nix-prefetch-url", "--name", plugin.filename(), url], capture_output=True,
-        capture_output=True,
     )
     if not res.stdout:
         raise IOError(f"nix-prefetch-url {plugin} failed: {res.stderr.decode('utf-8')}")
@@ -125,33 +169,49 @@ def write_packages(outfile, plugins, build):
     outfile.write("{callPackage}:\n{")
 
     for plugin in plugins:
+        src_url = plugin.download_url(build, deref=True)
+        src_ext = os.path.splitext(src_url)[-1]
+
         try:
-            sha = prefetch(plugin, build)
+            sha = prefetch(plugin, build, src_url)
         except IOError as e:
             print(e, file=sys.stderr)
             continue
+
+        build_inputs = []
+        if src_ext == ".zip":
+            build_inputs.append("unzip")
+
         # TODO: Dependencies are provided as package IDs but refer to both
         # internal and external plugins; need to find some way to resolve them
         requires = []
         # TODO: Licenses are actually on the website, but aren't provided in the API
-        license = "lib.license.free"
+        license = "lib.licenses.free"
+
+        call_args = [str(builder), "fetchurl", "lib"]
+        for binput in build_inputs:
+            call_args.append(binput)
 
         outfile.write(
             f"""
-  {plugin.packagename()} = callPackage ({{ {builder}, fetchurl, lib }}): {builder} {{
+  {plugin.packagename()} = callPackage ({{ {", ".join(sorted(call_args))} }}: {builder} {{
     pname = "{plugin.slug}";
     plugname = "{plugin.name}";
     plugid = "{plugin.id}";
     version = "{plugin.version}";
     src = fetchurl {{
-      url = "{plugin.download_url(build)}";
+      url = "{src_url}";
       sha256 = "{sha}";
-      name = "{plugin.filename()}";
+      name = "{plugin.filename()}{src_ext}";
     }};
-    packageRequires = {requires};
+    buildInputs = [ {" ".join(build_inputs)} ];
+    packageRequires = [ {" ".join(requires)} ];
     meta = {{
-      {f'homepage = {plugin.url};' if plugin.url else ''}
+      homepage = "{plugin.url or ""}";
       license = {license};
+      description = ''
+        {plugin.description()}
+      '';
     }};
   }}) {{}};
 """
@@ -179,9 +239,12 @@ def main():
     build = Build(args.package)
     plugins = list_plugins(build)
     plugins.sort(key=lambda p: p.slug)
+    deduplicate(plugins)
+
     if args.number:
         plugins = plugins[: args.number]
 
+    print(f"Generating packages for {len(plugins)} plugins", file=sys.stderr)
     if not args.out:
         write_packages(sys.stdout, plugins, build)
     else:

pkgs/applications/editors/jetbrains/common-plugins.nix

@@ -4,7 +4,7 @@ self:
 
 let
 
-  commonBuild = import ../../../build-support/jetbrains/plugin.nix {
+  commonBuild = import ../../../build-support/jetbrains/plugin-old.nix {
     inherit lib stdenv fetchzip;
     jetbrainsPlatforms = [
       "clion"

pkgs/applications/editors/jetbrains/idea-plugins.nix

@@ -1,19 +1,19 @@
-{ lib, stdenv, fetchzip }:
+{ lib, stdenv, variant }:
 
 self:
 
 let
 
   ideaBuild = import ../../../build-support/jetbrains/plugin.nix {
-    inherit lib stdenv fetchzip;
+    inherit lib stdenv variant;
     jetbrainsPlatforms = [ "idea-community" "idea-ultimate" ];
   };
 
   generateIdea = lib.makeOverridable ({
-    idea ? ./manual-idea-packages.nix
+    generated ? ./idea-generated.nix
   }: let
 
-    imported = import idea {
+    imported = import generated {
       inherit (self) callPackage;
     };
 
@@ -26,4 +26,3 @@ let
   in ideaPlugins // { inherit ideaBuild; });
 
 in generateIdea { }
-

pkgs/applications/editors/jetbrains/wrapper.nix

@@ -21,11 +21,13 @@ in
 assert assertMsg (length badPlugins == 0) errorMsg;
 
 appendToName "with-plugins" (package.overrideAttrs (oldAttrs: {
-  passthru = { inherit plugins; };
+  inherit plugins;
-  # TODO: Purely aesthetics, but link the plugin to its name instead of hash-name-version
+  # TODO: Remove version from directory name
   installPhase = oldAttrs.installPhase + ''
     for plugin in $plugins; do
-      ln -s "$plugin" "$out/$name/plugins/$(basename $plugin)"
+      local dirname=$(basename "$plugin")
+      dirname=''${dirname:33}
+      ln -s "$plugin" "$out/$name/plugins/$dirname"
     done
   '';
 }))

pkgs/build-support/jetbrains/plugin-old.nix

@@ -0,0 +1,33 @@
+{ lib
+, stdenv
+, fetchzip
+, jetbrainsPlatforms
+}:
+
+{ pluginId
+, pname
+, version
+, versionId
+, sha256
+, filename ? "${pname}-${version}.zip"
+}:
+
+stdenv.mkDerivation {
+  inherit pname version;
+
+  src = fetchzip {
+    inherit sha256;
+    url = "https://plugins.jetbrains.com/files/${toString pluginId}/${toString versionId}/${filename}";
+  };
+
+  passthru = { inherit jetbrainsPlatforms; };
+
+  installPhase = ''
+    mkdir $out
+    cp -r * $out/
+  '';
+
+  meta = {
+    homepage = "https://plugins.jetbrains.com/plugin/${pluginId}-${lib.toLower pname}";
+  };
+}

pkgs/build-support/jetbrains/plugin.nix

@@ -1,33 +1,71 @@
 { lib
 , stdenv
-, fetchzip
+, variant
 , jetbrainsPlatforms
 }:
 
-{ pluginId
+with lib;
-, pname
-, version
-, versionId
-, sha256
-, filename ? "${pname}-${version}.zip"
-}:
 
-stdenv.mkDerivation {
+{ pname
+, version
+
+, plugname
+, plugid
+
+, buildInputs ? []
+, packageRequires ? []
+, meta ? {}
+
+, ...
+}@args:
+
+let
+
+  defaultMeta = {
+    broken = false;
+    platforms = variant.meta.platforms;
+  } // optionalAttrs ((args.src.meta.homepage or "") != "") {
+    homepage = args.src.meta.homepage;
+  } // optionalAttrs ((args.src.meta.description or "") != "") {
+    description = args.src.meta.description;
+  };
+
+in
+
+stdenv.mkDerivation ({
   inherit pname version;
 
-  src = fetchzip {
+  unpackCmd = ''
-    inherit sha256;
+    case "$curSrc" in
-    url = "https://plugins.jetbrains.com/files/${toString pluginId}/${toString versionId}/${filename}";
+      *.jar)
-  };
+        # don't unpack; keep original source filename without the hash
-
+        local filename=$(basename "$curSrc")
-  passthru = { inherit jetbrainsPlatforms; };
+        filename="''${filename:33}"
-
+        cp $curSrc $filename
-  installPhase = ''
+        chmod +w $filename
-    mkdir $out
+        sourceRoot="."
-    cp -r * $out/
+        ;;
+      *)
+        _defaultUnpack "$curSrc"
+        ;;
+    esac
   '';
 
-  meta = {
+  # FIXME: Entirely possible this isn't correct for niche plugins;
-    homepage = "https://plugins.jetbrains.com/plugin/${pluginId}-${lib.toLower pname}";
+  # at the very least there are some plugins that come with JS
-  };
+  installPhase = ''
+    mkdir -p "$out/lib"
+    find -iname '*.jar' -exec cp {} "$out/lib/" \;
+  '';
+
+  buildInputs = [ ] ++ packageRequires ++ buildInputs;
+  propagatedBuildInputs = packageRequires;
+
+  passthru = { inherit jetbrainsPlatforms plugid plugname; };
+
+  doCheck = false;
+
+  meta = defaultMeta // meta;
 }
+
+// removeAttrs args [ "buildInputs" "packageRequires" "meta" ])

pkgs/top-level/jetbrains-plugins.nix

@@ -13,7 +13,7 @@ let
   };
 
   mkIdeaPlugins = import ../applications/editors/jetbrains/idea-plugins.nix {
-    inherit lib stdenv fetchzip;
+    inherit lib stdenv variant;
   };
 
   jetbrainsWithPlugins = import ../applications/editors/jetbrains/wrapper.nix {