jetbrains: generate IDEA packages

The main script only does a single program, and requires being fed the
name. This is a wrapper to get those names.

pkgs/applications/editors/jetbrains/common-plugins.nix

@@ -1,41 +0,0 @@
-{ lib, stdenv, fetchzip }:
-
-self:
-
-let
-
-  commonBuild = import ../../../build-support/jetbrains/plugin.nix {
-    inherit lib stdenv fetchzip;
-    jetbrainsPlatforms = [
-      "clion"
-      "datagrip"
-      "goland"
-      "idea-community"
-      "idea-ultimate"
-      "phpstorm"
-      "pycharm-community"
-      "pycharm-professional"
-      "rider"
-      "ruby-mine"
-      "webstorm"
-    ];
-  };
-
-  generateCommon = lib.makeOverridable ({
-    common ? ./manual-common-packages.nix
-  }: let
-
-    imported = import common {
-      inherit (self) callPackage;
-    };
-
-    super = imported;
-
-    overrides = { };
-
-    jetbrainsPlugins = super // overrides;
-
-  in jetbrainsPlugins // { inherit commonBuild; });
-
-in generateCommon { }
-

pkgs/applications/editors/jetbrains/idea-plugins.nix

@@ -10,7 +10,7 @@ let
   };
 
   generateIdea = lib.makeOverridable ({
-    idea ? ./manual-idea-packages.nix
+    idea ? ./idea-generated.nix
   }: let
 
     imported = import idea {

pkgs/applications/editors/jetbrains/manual-common-packages.nix

@@ -1,33 +0,0 @@
-{ callPackage }:
-{
-  ideavim = callPackage
-    ({ commonBuild }: commonBuild {
-      pname = "IdeaVim";
-      version = "0.57";
-      pluginId = 164;
-      versionId = 85009;
-      sha256 = "1rwfwj0b0nwi7jxhzxk1r0xc190nf4i3b59i0zknpmgb4yc5clzw";
-    })
-    { };
-
-  checkstyle-idea = callPackage
-    ({ commonBuild }: commonBuild {
-      pname = "CheckStyle-IDEA";
-      version = "5.42.0";
-      pluginId = 1065;
-      versionId = 95757;
-      sha256 = "0sji3649n5zz84dlidqaklipq6vaiafxsvg0gzy3j59mvkz6dk14";
-    })
-    { };
-
-  google-java-format = callPackage
-    ({ commonBuild }: commonBuild rec {
-      pname = "google-java-format";
-      version = "1.7.0.4";
-      pluginId = 8527;
-      versionId = 83164;
-      sha256 = "1pmnn1ksiv44kdga53gi3psrm2sva4bqrxizagbr0if2n0rrvgii";
-      filename = "${pname}.zip";
-    })
-    { };
-}

pkgs/applications/editors/jetbrains/manual-idea-packages.nix

@@ -1,12 +0,0 @@
-{ callPackage }:
-{
-  spring-assistant = callPackage
-    ({ ideaBuild }: ideaBuild {
-      pname = "intellij-spring-assistant";
-      version = "0.12.0";
-      pluginId = 10229;
-      versionId = 44968;
-      sha256 = "13cglywzhb4j0qj0bs2jwaz2k8pxrxalv35wgkmgkxr635bxmwsj";
-    })
-    { };
-}

pkgs/applications/editors/jetbrains/update-jetbrains

@@ -0,0 +1,12 @@
+#!/usr/bin/env nix-shell
+#!nix-shell --show-trace ./updater.nix -i bash
+
+set -e
+
+for app in $APPS; do
+  ./updater-jetbrains $app -o ${app%-[0-9.]*}-generated.nix
+done
+
+# Local Variables:
+# mode: sh
+# End:

pkgs/applications/editors/jetbrains/updater-jetbrains

@@ -7,6 +7,7 @@ import re
 import requests
 import subprocess as sp
 import sys
+import unicodedata
 import urllib
 
 from lxml import etree
@@ -27,10 +28,15 @@ PRODUCT_CODE = {
 }
 
 
+PACKAGE_RE = re.compile("[^0-9A-Za-z._-]")
+HTML_RE = re.compile("<[^>]+/?>")
+
+
 def to_slug(name):
     slug = name.replace(" ", "-").lstrip(".")
     for char in ",/;'\\<>:\"|!@#$%^&*()":
         slug = slug.replace(char, "")
+    slug = unicodedata.normalize("NFD", slug).encode("ascii", "ignore").decode("utf-8")
     return slug
 
 
@@ -59,18 +65,15 @@ class Build:
         return self.code + "-" + self.version
 
 
-PACKAGE_RE = re.compile("[^0-9A-Za-z._-]")
-HTML_RE = re.compile("<[^>]+/?>")
-
-
 class Plugin:
-    def __init__(self, data, category=None):
+    def __init__(self, data, build, category=None):
+        self.build = build
         self.category = category
         self.name = data.find("name").text
-        self.id = data.find("id").text
+        self.xml_id = data.find("id").text
         self._description = data.find("description").text
         self.url = data.get("url") or data.find("vendor").get("url")
-        self.version = data.find("version").text
+        self.version = data.find("version").text.replace(" ", "-")
         self.slug = to_slug(self.name)
         self.orig_slug = self.slug
 
@@ -84,7 +87,7 @@ class Plugin:
     def description(self):
         return re.sub(HTML_RE, "", self._description or "").strip()
 
-    def download_url(self, build, deref=True):
+    def get_download_url(self, deref=True):
         """
         Provides the ZIP download URL for this plugin.
 
@@ -94,18 +97,52 @@ class Plugin:
         (which it is by default). However, this comes at the cost of requiring
         an HTTP request.
         """
-        id = urllib.parse.quote(self.id)
+        id = urllib.parse.quote(self.xml_id)
-        url = f"https://plugins.jetbrains.com/pluginManager?action=download&id={id}&build={build}"
+        url = f"https://plugins.jetbrains.com/pluginManager?action=download&id={id}&build={self.build}"
         if deref:
             res = requests.get(url, allow_redirects=not deref)
             url = "https://plugins.jetbrains.com" + re.sub(
                 "\?.*$", "", res.headers["location"]
             )
+            self.jetbrains_url = url
             if url.endswith("external"):
                 res = requests.get(url, allow_redirects=not deref)
                 url = res.headers["location"]
         return url
 
+    def fetch_external(self, update_only=False):
+        """
+        Performs network calls to update this plugin with information that
+        cannot be performed from the public XML API.
+
+        Additional attributes provided after this method:
+
+          download_url : the plugin download location
+          sha          : the SHA256 of the download source
+
+        If update_only is true, a full update is performed, also providing:
+
+          id          : the plugin integer ID
+          license_url : the plugin license URL
+          license     : the Nixpkgs license attribute
+        """
+        self.download_url = self.get_download_url(deref=True)
+        self.sha = prefetch(self, self.build, self.download_url)
+
+        if update_only:
+            return
+
+        self.id = self.jetbrains_url.split("/")[4]
+        res = requests.get(
+            f"https://plugins.jetbrains.com/api/plugins/{self.id}"
+        ).json()
+        try:
+            self.url = self.url or res["urls"]["sourceCodeUrl"]
+        except KeyError:
+            pass
+        self.license_url = res["urls"]["licenseUrl"]
+        self.license = translate_license(self.license_url, fallback=self.url)
+
     def packagename(self):
         slug = re.sub(PACKAGE_RE, "", self.slug.lower()).replace(".", "-")
         if slug[0] in "1234567890":
@@ -128,16 +165,16 @@ def list_plugins(build):
     https://plugins.jetbrains.com/docs/marketplace/plugins-list.html
     """
     resp = requests.get(f"https://plugins.jetbrains.com/plugins/list/?build={build}")
-    return parse_repository(resp.content)
+    return parse_repository(resp.content, build)
 
 
-def parse_repository(content):
+def parse_repository(content, build):
     tree = etree.XML(content)
     plugins = []
     for cat in tree.findall("category"):
         cat_name = cat.get("name")
         for plugin in cat.findall("idea-plugin"):
-            plugins.append(Plugin(plugin, cat_name))
+            plugins.append(Plugin(plugin, build, cat_name))
     return plugins
 
 
@@ -155,28 +192,101 @@ def deduplicate(plugins):
 
 def prefetch(plugin, build, url=None):
     if not url:
-        url = plugin.download_url(build)
+        url = plugin.download_url or plugin.get_download_url()
     res = sp.run(
         ["nix-prefetch-url", "--name", plugin.filename(), url], capture_output=True,
     )
     if not res.stdout:
-        raise IOError(f"nix-prefetch-url {plugin} failed: {res.stderr.decode('utf-8')}")
+        raise IOError(
-    return res.stdout.decode("utf-8").strip()
+            f"nix-prefetch-url {plugin} failed: {res.stderr.decode('unicode-escape')}"
+        )
+    return res.stdout.decode("unicode-escape").strip()
 
 
-def write_packages(outfile, plugins, build):
+def custom_license(short, full, url, free=False):
-    builder = build.builder()
+    return f"""{{
+        shortName = "{short}";
+        fullName = "{full}";
+        url = "{url}";
+        free = {"true" if free else "false"};
+      }}"""
+
+
+def arr(url):
+    return custom_license("allrightsreserved", "All Rights Reserved", url)
+
+
+def translate_license(url, fallback=""):
+    license = url.lower()
+    if license == "":
+        print(f"no license for {fallback}", file=sys.stderr)
+        return arr(fallback)
+    # Common (license) hosts
+    elif "github.com" in license or "raw.githubusercontent.com" in license:
+        try:
+            owner, repo = url.split("/")[3:5]
+        except ValueError:
+            print(f"no license metadata for {url}", file=sys.stderr)
+            return arr(url)
+        res = requests.get(
+            f"https://api.github.com/repos/{owner}/{repo}",
+            headers={"Accept": "application/vnd.github.v3+json"},
+        ).json()
+        try:
+            return translate_license(res["license"]["key"])
+        except (KeyError, TypeError):
+            print(f"no license metadata for {url}", file=sys.stderr)
+            return arr(url)
+    elif "opensource.org" in license:
+        os_license = license.rstrip("/").split("/")[-1]
+        if os_license == "alphabetical":
+            # Doesn't actually have a license, it's the listing page
+            return arr(fallback)
+        return translate_license(os_license)
+    # Actual translations now
+    elif "apache.org/licenses/license-2.0" in license or "apache-2.0" in license:
+        return "lib.licenses.asl20"
+    elif "artistic-2" in license:
+        return "lib.licenses.artistic2"
+    elif "bsd-2-clause" in license:
+        return "lib.licenses.bsd2"
+    elif "bsd-3-clause" in license:
+        return "lib.licenses.bsd3"
+    elif "eclipse.org/legal/epl-2.0" in license:
+        return "lib.licenses.epl20"
+    elif "gpl-3.0" in license:
+        return "lib.licenses.gpl3Only"
+    elif "mit" in license:
+        return "lib.licenses.mit"
+    elif "osd" in license:
+        return "lib.licenses.free"
+    elif "other" == license:
+        return arr(fallback)
+    # Custom known licenses
+    elif "plugins.jetbrains.com/legal/terms-of-use" in license:
+        return custom_license(
+            "jetbrains", "Jetbrains Plugin Marketplace Agreement", license
+        )
+    # Fallback
+    else:
+        print(f"unrecognised license {license}", file=sys.stderr)
+        return arr(license)
+
+
+def write_packages(outfile, plugins):
+    builder = plugins[0].build.builder() or ""
     outfile.write("{callPackage}:\n{")
 
-    for plugin in plugins:
+    for i, plugin in enumerate(plugins):
-        src_url = plugin.download_url(build, deref=True)
+        print(f"{i:04} {plugin.packagename()}")
-        src_ext = os.path.splitext(src_url)[-1]
-
         try:
-            sha = prefetch(plugin, build, src_url)
+            plugin.fetch_external()
         except IOError as e:
             print(e, file=sys.stderr)
             continue
+        src_url = plugin.download_url
+        src_ext = os.path.splitext(src_url)[-1]
+        sha = plugin.sha
 
         build_inputs = []
         if src_ext == ".zip":
@@ -186,7 +296,7 @@ def write_packages(outfile, plugins, build):
         # internal and external plugins; need to find some way to resolve them
         requires = []
         # TODO: Licenses are actually on the website, but aren't provided in the API
-        license = "lib.licenses.free"
+        license = plugin.license
 
         call_args = [str(builder), "fetchurl", "lib"]
         for binput in build_inputs:
@@ -197,7 +307,7 @@ def write_packages(outfile, plugins, build):
   {plugin.packagename()} = callPackage ({{ {", ".join(sorted(call_args))} }}: {builder} {{
     pname = "{plugin.slug}";
     plugname = "{plugin.name}";
-    plugid = "{plugin.id}";
+    plugid = "{plugin.xml_id}";
     version = "{plugin.version}";
     src = fetchurl {{
       url = "{src_url}";
@@ -227,6 +337,7 @@ def main():
     parser.add_argument(
         "-o", "--out", type=str, help="File to write plugins to",
     )
+    parser.add_argument("-O", "--offset", type=int, help="Offset number of packages")
     parser.add_argument(
         "package",
         metavar="PACKAGE",
@@ -241,15 +352,17 @@ def main():
     plugins.sort(key=lambda p: p.slug)
     deduplicate(plugins)
 
+    if args.offset:
+        plugins = plugins[args.offset :]
     if args.number:
         plugins = plugins[: args.number]
 
     print(f"Generating packages for {len(plugins)} plugins", file=sys.stderr)
     if not args.out:
-        write_packages(sys.stdout, plugins, build)
+        write_packages(sys.stdout, plugins)
     else:
         with open(args.out, "w") as f:
-            write_packages(f, plugins, build)
+            write_packages(f, plugins)
 
 
 main()

pkgs/applications/editors/jetbrains/updater.nix

@@ -0,0 +1,26 @@
+{ pkgs ? import <nixpkgs> { } }:
+let
+  inherit (pkgs.stdenv) lib;
+in
+pkgs.mkShell {
+  buildInputs = [
+    pkgs.bash
+  ];
+
+  APPS = lib.mapAttrsToList
+    (k: v: v.name)
+    {
+      inherit (pkgs.jetbrains)
+        clion
+        datagrip
+        goland
+        idea-community
+        idea-ultimate
+        phpstorm
+        pycharm-community
+        pycharm-professional
+        rider
+        ruby-mine
+        webstorm;
+    };
+}

pkgs/applications/editors/jetbrains/wrapper.nix

@@ -1,5 +1,4 @@
-{ lib
+{ lib, makeWrapper, runCommand }: self:
-}: self:
 
 with lib;
 
@@ -20,12 +19,31 @@ in
 
 assert assertMsg (length badPlugins == 0) errorMsg;
 
-appendToName "with-plugins" (package.overrideAttrs (oldAttrs: {
+runCommand
-  passthru = { inherit plugins; };
+  (appendToName "with-plugins" package).name
-  # TODO: Purely aesthetics, but link the plugin to its name instead of hash-name-version
+{
-  installPhase = oldAttrs.installPhase + ''
+  nativeBuildInputs = [ package makeWrapper ];
-    for plugin in $plugins; do
+  inherit package plugins;
-      ln -s "$plugin" "$out/$name/plugins/$(basename $plugin)"
+  packageName = package.name;
+
+  preferLocalBuild = true;
+  allowSubstitutes = false;
+
+} ''
+  mkdir -p $out/$packageName/plugins
+  for dir in $package/*; do
+    cp -r $dir $out/
   done
-  '';
+
-}))
+  # Install plugins
+  for plugin in $plugins; do
+    local pluginName=$(basename $plugin)
+    pluginName=''${pluginName#*-}
+    pluginName=''${pluginName%-[0-9.]*}
+    ln -s $plugin $out/$packageName/plugins/$pluginName
+  done
+
+  # Fix up wrapper
+  substituteInPlace $out/bin/* \
+    --replace "$package" "$out"
+''

pkgs/build-support/jetbrains/plugin.nix

@@ -4,30 +4,38 @@
 , jetbrainsPlatforms
 }:
 
-{ pluginId
+{ plugid
 , pname
 , version
-, versionId
+, ...
-, sha256
+}@args:
-, filename ? "${pname}-${version}.zip"
-}:
 
-stdenv.mkDerivation {
+let
-  inherit pname version;
 
-  src = fetchzip {
+  defaultMeta = {
-    inherit sha256;
+    broken = false;
-    url = "https://plugins.jetbrains.com/files/${toString pluginId}/${toString versionId}/${filename}";
+  } // lib.optionalAttrs ((args.src.meta.homepage or "") != "") {
+    homepage = args.src.meta.homepage;
+  } // lib.optionalAttrs ((args.src.meta.description or "") != "") {
+    description = args.src.meta.description;
+  } // lib.optionalAttrs ((args.src.meta.license or {}) != {}) {
+    license = args.src.meta.license;
   };
 
+in
+
+stdenv.mkDerivation (args // {
   passthru = { inherit jetbrainsPlatforms; };
 
+  dontUnpack = lib.any (lib.hasSuffix ".jar") args.src.urls;
+
   installPhase = ''
     mkdir $out
     cp -r * $out/
   '';
 
   meta = {
-    homepage = "https://plugins.jetbrains.com/plugin/${pluginId}-${lib.toLower pname}";
+    inherit (args.meta) license description;
+    homepage = if (args.meta.homepage == "") then null else args.meta.homepage;
   };
-}
+})

pkgs/top-level/all-packages.nix

@@ -55,7 +55,7 @@ rec {
   # A functional Jetbrains IDE-with-plugins package set.
   jetbrains = pkgs.dontRecurseIntoAttrs rec {
     jetbrainsPluginsFor = variant: import ../top-level/jetbrains-plugins.nix {
-      inherit (pkgs) lib newScope stdenv fetchzip;
+      inherit (pkgs) lib newScope stdenv fetchzip makeWrapper runCommand;
       inherit variant;
     };
 

pkgs/top-level/jetbrains-plugins.nix

@@ -2,29 +2,25 @@
 , newScope
 , stdenv
 , fetchzip
+, makeWrapper
+, runCommand
 
 , variant
 }:
 
 let
 
-  mkJetbrainsPlugins = import ../applications/editors/jetbrains/common-plugins.nix {
-    inherit lib stdenv fetchzip;
-  };
-
   mkIdeaPlugins = import ../applications/editors/jetbrains/idea-plugins.nix {
     inherit lib stdenv fetchzip;
   };
 
   jetbrainsWithPlugins = import ../applications/editors/jetbrains/wrapper.nix {
-    inherit lib;
+    inherit lib makeWrapper runCommand;
   };
 
 in lib.makeScope newScope (self: lib.makeOverridable ({
-  jetbrainsPlugins ? mkJetbrainsPlugins self
+  ideaPlugins ? mkIdeaPlugins self
-  , ideaPlugins ? mkIdeaPlugins self
 }: ({ }
-  // jetbrainsPlugins // { inherit jetbrainsPlugins; }
   // ideaPlugins // { inherit ideaPlugins; }
   // {
     inherit variant;