jetbrains: update IDEA for generated plugins

bin/update-jetbrains-plugins.py

@@ -0,0 +1,255 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i python3 -p python3Packages.lxml python3Packages.requests
+
+import argparse
+import os
+import re
+import requests
+import subprocess as sp
+import sys
+import urllib
+
+from lxml import etree
+
+# https://plugins.jetbrains.com/docs/marketplace/product-codes.html
+PRODUCT_CODE = {
+    "clion": "CL",
+    "datagrip": "DB",
+    "goland": "GO",
+    "idea-community": "IC",
+    "idea-ultimate": "IU",
+    "phpstorm": "PS",
+    "pycharm-community": "PC",
+    "pycharm-professional": "PY",
+    "rider": "RD",
+    "ruby-mine": "RM",
+    "webstorm": "WS",
+}
+
+
+def to_slug(name):
+    slug = name.replace(" ", "-").lstrip(".")
+    for char in ",/;'\\<>:\"|!@#$%^&*()":
+        slug = slug.replace(char, "")
+    return slug
+
+
+class Build:
+    """
+    Transforms a Nixpkgs derivation name into a Jetbrains product code. For
+    example:
+
+    idea-community-2019.3.2 -> IC-193.2
+    """
+
+    def __init__(self, name):
+        m = re.search("([0-9]+\.?)+$", name)
+        version = m.group(0)
+        code = PRODUCT_CODE[name.replace("-" + version, "")]
+        vparts = version.split(".")
+        version = vparts[0][-2:] + vparts[1] + "." + vparts[2]
+        self.code = code
+        self.version = version
+        self.package = name.split("-")[0]
+
+    def builder(self):
+        return self.package + "Build"
+
+    def __repr__(self):
+        return self.code + "-" + self.version
+
+
+PACKAGE_RE = re.compile("[^0-9A-Za-z._-]")
+HTML_RE = re.compile("<[^>]+/?>")
+
+
+class Plugin:
+    def __init__(self, data, category=None):
+        self.category = category
+        self.name = data.find("name").text
+        self.id = data.find("id").text
+        self._description = data.find("description").text
+        self.url = data.get("url") or data.find("vendor").get("url")
+        self.version = data.find("version").text
+        self.slug = to_slug(self.name)
+        self.orig_slug = self.slug
+
+        self.depends = []
+        for depend in data.findall("depends"):
+            self.depends.append(depend.text)
+
+    def __repr__(self):
+        return f"<Plugin '{self.name}' {self.version}>"
+
+    def description(self):
+        return re.sub(HTML_RE, "", self._description or "").strip()
+
+    def download_url(self, build, deref=True):
+        """
+        Provides the ZIP download URL for this plugin.
+
+        The trivial URL fetches the latest version through a redirect for some
+        build code and provides no locking to a version for the URL. To fetch
+        a stable URL that can be used as a package source, deref must be set
+        (which it is by default). However, this comes at the cost of requiring
+        an HTTP request.
+        """
+        id = urllib.parse.quote(self.id)
+        url = f"https://plugins.jetbrains.com/pluginManager?action=download&id={id}&build={build}"
+        if deref:
+            res = requests.get(url, allow_redirects=not deref)
+            url = "https://plugins.jetbrains.com" + re.sub(
+                "\?.*$", "", res.headers["location"]
+            )
+            if url.endswith("external"):
+                res = requests.get(url, allow_redirects=not deref)
+                url = res.headers["location"]
+        return url
+
+    def packagename(self):
+        slug = re.sub(PACKAGE_RE, "", self.slug.lower()).replace(".", "-")
+        if slug[0] in "1234567890":
+            return "_" + slug
+        else:
+            return slug
+
+    def filename(self):
+        """
+        Returns this plugin's filename without an extension. Rely on the
+        download URL to know the extension.
+        """
+        return f"{self.slug}-{self.version}"
+
+
+def list_plugins(build):
+    """
+    Lists all plugins for the specified build code.
+
+    https://plugins.jetbrains.com/docs/marketplace/plugins-list.html
+    """
+    resp = requests.get(f"https://plugins.jetbrains.com/plugins/list/?build={build}")
+    return parse_repository(resp.content)
+
+
+def parse_repository(content):
+    tree = etree.XML(content)
+    plugins = []
+    for cat in tree.findall("category"):
+        cat_name = cat.get("name")
+        for plugin in cat.findall("idea-plugin"):
+            plugins.append(Plugin(plugin, cat_name))
+    return plugins
+
+
+def deduplicate(plugins):
+    """
+    Ensures that the plugin list has unique slugs. Modifies the list in-place.
+    """
+    prev = plugins[0]
+    for plugin in plugins[1:]:
+        if plugin.orig_slug == prev.orig_slug:
+            prev.slug = prev.orig_slug + "-" + prev.version.replace(".", "_")
+            plugin.slug = plugin.orig_slug + "-" + plugin.version.replace(".", "_")
+        prev = plugin
+
+
+def prefetch(plugin, build, url=None):
+    if not url:
+        url = plugin.download_url(build)
+    res = sp.run(
+        ["nix-prefetch-url", "--name", plugin.filename(), url], capture_output=True,
+    )
+    if not res.stdout:
+        raise IOError(f"nix-prefetch-url {plugin} failed: {res.stderr.decode('utf-8')}")
+    return res.stdout.decode("utf-8").strip()
+
+
+def write_packages(outfile, plugins, build):
+    builder = build.builder()
+    outfile.write("{callPackage}:\n{")
+
+    for plugin in plugins:
+        src_url = plugin.download_url(build, deref=True)
+        src_ext = os.path.splitext(src_url)[-1]
+
+        try:
+            sha = prefetch(plugin, build, src_url)
+        except IOError as e:
+            print(e, file=sys.stderr)
+            continue
+
+        build_inputs = []
+        if src_ext == ".zip":
+            build_inputs.append("unzip")
+
+        # TODO: Dependencies are provided as package IDs but refer to both
+        # internal and external plugins; need to find some way to resolve them
+        requires = []
+        # TODO: Licenses are actually on the website, but aren't provided in the API
+        license = "lib.licenses.free"
+
+        call_args = [str(builder), "fetchurl", "lib"]
+        for binput in build_inputs:
+            call_args.append(binput)
+
+        outfile.write(
+            f"""
+  {plugin.packagename()} = callPackage ({{ {", ".join(sorted(call_args))} }}: {builder} {{
+    pname = "{plugin.slug}";
+    plugname = "{plugin.name}";
+    plugid = "{plugin.id}";
+    version = "{plugin.version}";
+    src = fetchurl {{
+      url = "{src_url}";
+      sha256 = "{sha}";
+      name = "{plugin.filename()}{src_ext}";
+    }};
+    buildInputs = [ {" ".join(build_inputs)} ];
+    packageRequires = [ {" ".join(requires)} ];
+    meta = {{
+      homepage = "{plugin.url or ""}";
+      license = {license};
+      description = ''
+        {plugin.description()}
+      '';
+    }};
+  }}) {{}};
+"""
+        )
+    outfile.write("}\n")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-n", "--number", type=int, help="Limit the number of packages to write",
+    )
+    parser.add_argument(
+        "-o", "--out", type=str, help="File to write plugins to",
+    )
+    parser.add_argument(
+        "package",
+        metavar="PACKAGE",
+        type=str,
+        help="The Nixpkgs package name (inc. version)",
+    )
+
+    args = parser.parse_args()
+
+    build = Build(args.package)
+    plugins = list_plugins(build)
+    plugins.sort(key=lambda p: p.slug)
+    deduplicate(plugins)
+
+    if args.number:
+        plugins = plugins[: args.number]
+
+    print(f"Generating packages for {len(plugins)} plugins", file=sys.stderr)
+    if not args.out:
+        write_packages(sys.stdout, plugins, build)
+    else:
+        with open(args.out, "w") as f:
+            write_packages(f, plugins, build)
+
+
+main()

pkgs/applications/editors/jetbrains/common-plugins.nix

@@ -4,7 +4,7 @@ self:
 
 let
 
-  commonBuild = import ../../../build-support/jetbrains/plugin.nix {
+  commonBuild = import ../../../build-support/jetbrains/plugin-old.nix {
     inherit lib stdenv fetchzip;
     jetbrainsPlatforms = [
       "clion"

pkgs/applications/editors/jetbrains/idea-plugins.nix

@@ -1,19 +1,19 @@
-{ lib, stdenv, fetchzip }:
+{ lib, stdenv, variant }:
 
 self:
 
 let
 
   ideaBuild = import ../../../build-support/jetbrains/plugin.nix {
-    inherit lib stdenv fetchzip;
+    inherit lib stdenv variant;
     jetbrainsPlatforms = [ "idea-community" "idea-ultimate" ];
   };
 
   generateIdea = lib.makeOverridable ({
-    idea ? ./manual-idea-packages.nix
+    generated ? ./idea-generated.nix
   }: let
 
-    imported = import idea {
+    imported = import generated {
       inherit (self) callPackage;
     };
 
@@ -26,4 +26,3 @@ let
   in ideaPlugins // { inherit ideaBuild; });
 
 in generateIdea { }
-

pkgs/applications/editors/jetbrains/wrapper.nix

@@ -21,11 +21,13 @@ in
 assert assertMsg (length badPlugins == 0) errorMsg;
 
 appendToName "with-plugins" (package.overrideAttrs (oldAttrs: {
-  passthru = { inherit plugins; };
+  inherit plugins;
-  # TODO: Purely aesthetics, but link the plugin to its name instead of hash-name-version
+  # TODO: Remove version from directory name
   installPhase = oldAttrs.installPhase + ''
     for plugin in $plugins; do
-      ln -s "$plugin" "$out/$name/plugins/$(basename $plugin)"
+      local dirname=$(basename "$plugin")
+      dirname=''${dirname:33}
+      ln -s "$plugin" "$out/$name/plugins/$dirname"
     done
   '';
 }))

pkgs/build-support/jetbrains/plugin-old.nix

@@ -0,0 +1,33 @@
+{ lib
+, stdenv
+, fetchzip
+, jetbrainsPlatforms
+}:
+
+{ pluginId
+, pname
+, version
+, versionId
+, sha256
+, filename ? "${pname}-${version}.zip"
+}:
+
+stdenv.mkDerivation {
+  inherit pname version;
+
+  src = fetchzip {
+    inherit sha256;
+    url = "https://plugins.jetbrains.com/files/${toString pluginId}/${toString versionId}/${filename}";
+  };
+
+  passthru = { inherit jetbrainsPlatforms; };
+
+  installPhase = ''
+    mkdir $out
+    cp -r * $out/
+  '';
+
+  meta = {
+    homepage = "https://plugins.jetbrains.com/plugin/${pluginId}-${lib.toLower pname}";
+  };
+}

pkgs/build-support/jetbrains/plugin.nix

@@ -1,33 +1,71 @@
 { lib
 , stdenv
-, fetchzip
+, variant
 , jetbrainsPlatforms
 }:
 
-{ pluginId
+with lib;
-, pname
-, version
-, versionId
-, sha256
-, filename ? "${pname}-${version}.zip"
-}:
 
-stdenv.mkDerivation {
+{ pname
+, version
+
+, plugname
+, plugid
+
+, buildInputs ? []
+, packageRequires ? []
+, meta ? {}
+
+, ...
+}@args:
+
+let
+
+  defaultMeta = {
+    broken = false;
+    platforms = variant.meta.platforms;
+  } // optionalAttrs ((args.src.meta.homepage or "") != "") {
+    homepage = args.src.meta.homepage;
+  } // optionalAttrs ((args.src.meta.description or "") != "") {
+    description = args.src.meta.description;
+  };
+
+in
+
+stdenv.mkDerivation ({
   inherit pname version;
 
-  src = fetchzip {
+  unpackCmd = ''
-    inherit sha256;
+    case "$curSrc" in
-    url = "https://plugins.jetbrains.com/files/${toString pluginId}/${toString versionId}/${filename}";
+      *.jar)
-  };
+        # don't unpack; keep original source filename without the hash
-
+        local filename=$(basename "$curSrc")
-  passthru = { inherit jetbrainsPlatforms; };
+        filename="''${filename:33}"
-
+        cp $curSrc $filename
-  installPhase = ''
+        chmod +w $filename
-    mkdir $out
+        sourceRoot="."
-    cp -r * $out/
+        ;;
+      *)
+        _defaultUnpack "$curSrc"
+        ;;
+    esac
   '';
 
-  meta = {
+  # FIXME: Entirely possible this isn't correct for niche plugins;
-    homepage = "https://plugins.jetbrains.com/plugin/${pluginId}-${lib.toLower pname}";
+  # at the very least there are some plugins that come with JS
-  };
+  installPhase = ''
+    mkdir -p "$out/lib"
+    find -iname '*.jar' -exec cp {} "$out/lib/" \;
+  '';
+
+  buildInputs = [ ] ++ packageRequires ++ buildInputs;
+  propagatedBuildInputs = packageRequires;
+
+  passthru = { inherit jetbrainsPlatforms plugid plugname; };
+
+  doCheck = false;
+
+  meta = defaultMeta // meta;
 }
+
+// removeAttrs args [ "buildInputs" "packageRequires" "meta" ])

pkgs/top-level/jetbrains-plugins.nix

@@ -13,7 +13,7 @@ let
   };
 
   mkIdeaPlugins = import ../applications/editors/jetbrains/idea-plugins.nix {
-    inherit lib stdenv fetchzip;
+    inherit lib stdenv variant;
   };
 
   jetbrainsWithPlugins = import ../applications/editors/jetbrains/wrapper.nix {