From bb49706b0223a8bc5bd7639d9af7e379025255a7 Mon Sep 17 00:00:00 2001 From: xeals Date: Mon, 19 Oct 2020 19:09:03 +1100 Subject: [PATCH] jetbrains: add plugin generation script --- bin/update-jetbrains-plugins.py | 255 ++++++++++++++++++++++++++++++++ 1 file changed, 255 insertions(+) create mode 100755 bin/update-jetbrains-plugins.py diff --git a/bin/update-jetbrains-plugins.py b/bin/update-jetbrains-plugins.py new file mode 100755 index 0000000..115cdb4 --- /dev/null +++ b/bin/update-jetbrains-plugins.py @@ -0,0 +1,255 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i python3 -p python3Packages.lxml python3Packages.requests + +import argparse +import os +import re +import requests +import subprocess as sp +import sys +import urllib + +from lxml import etree + +# https://plugins.jetbrains.com/docs/marketplace/product-codes.html +PRODUCT_CODE = { + "clion": "CL", + "datagrip": "DB", + "goland": "GO", + "idea-community": "IC", + "idea-ultimate": "IU", + "phpstorm": "PS", + "pycharm-community": "PC", + "pycharm-professional": "PY", + "rider": "RD", + "ruby-mine": "RM", + "webstorm": "WS", +} + + +def to_slug(name): + slug = name.replace(" ", "-").lstrip(".") + for char in ",/;'\\<>:\"|!@#$%^&*()": + slug = slug.replace(char, "") + return slug + + +class Build: + """ + Transforms a Nixpkgs derivation name into a Jetbrains product code. For + example: + + idea-community-2019.3.2 -> IC-193.2 + """ + + def __init__(self, name): + m = re.search("([0-9]+\.?)+$", name) + version = m.group(0) + code = PRODUCT_CODE[name.replace("-" + version, "")] + vparts = version.split(".") + version = vparts[0][-2:] + vparts[1] + "." + vparts[2] + self.code = code + self.version = version + self.package = name.split("-")[0] + + def builder(self): + return self.package + "Build" + + def __repr__(self): + return self.code + "-" + self.version + + +PACKAGE_RE = re.compile("[^0-9A-Za-z._-]") +HTML_RE = re.compile("<[^>]+/?>") + + +class Plugin: + def __init__(self, data, category=None): + self.category = category + self.name = data.find("name").text + self.id = data.find("id").text + self._description = data.find("description").text + self.url = data.get("url") or data.find("vendor").get("url") + self.version = data.find("version").text + self.slug = to_slug(self.name) + self.orig_slug = self.slug + + self.depends = [] + for depend in data.findall("depends"): + self.depends.append(depend.text) + + def __repr__(self): + return f"" + + def description(self): + return re.sub(HTML_RE, "", self._description or "").strip() + + def download_url(self, build, deref=True): + """ + Provides the ZIP download URL for this plugin. + + The trivial URL fetches the latest version through a redirect for some + build code and provides no locking to a version for the URL. To fetch + a stable URL that can be used as a package source, deref must be set + (which it is by default). However, this comes at the cost of requiring + an HTTP request. + """ + id = urllib.parse.quote(self.id) + url = f"https://plugins.jetbrains.com/pluginManager?action=download&id={id}&build={build}" + if deref: + res = requests.get(url, allow_redirects=not deref) + url = "https://plugins.jetbrains.com" + re.sub( + "\?.*$", "", res.headers["location"] + ) + if url.endswith("external"): + res = requests.get(url, allow_redirects=not deref) + url = res.headers["location"] + return url + + def packagename(self): + slug = re.sub(PACKAGE_RE, "", self.slug.lower()).replace(".", "-") + if slug[0] in "1234567890": + return "_" + slug + else: + return slug + + def filename(self): + """ + Returns this plugin's filename without an extension. Rely on the + download URL to know the extension. + """ + return f"{self.slug}-{self.version}" + + +def list_plugins(build): + """ + Lists all plugins for the specified build code. + + https://plugins.jetbrains.com/docs/marketplace/plugins-list.html + """ + resp = requests.get(f"https://plugins.jetbrains.com/plugins/list/?build={build}") + return parse_repository(resp.content) + + +def parse_repository(content): + tree = etree.XML(content) + plugins = [] + for cat in tree.findall("category"): + cat_name = cat.get("name") + for plugin in cat.findall("idea-plugin"): + plugins.append(Plugin(plugin, cat_name)) + return plugins + + +def deduplicate(plugins): + """ + Ensures that the plugin list has unique slugs. Modifies the list in-place. + """ + prev = plugins[0] + for plugin in plugins[1:]: + if plugin.orig_slug == prev.orig_slug: + prev.slug = prev.orig_slug + "-" + prev.version.replace(".", "_") + plugin.slug = plugin.orig_slug + "-" + plugin.version.replace(".", "_") + prev = plugin + + +def prefetch(plugin, build, url=None): + if not url: + url = plugin.download_url(build) + res = sp.run( + ["nix-prefetch-url", "--name", plugin.filename(), url], capture_output=True, + ) + if not res.stdout: + raise IOError(f"nix-prefetch-url {plugin} failed: {res.stderr.decode('utf-8')}") + return res.stdout.decode("utf-8").strip() + + +def write_packages(outfile, plugins, build): + builder = build.builder() + outfile.write("{callPackage}:\n{") + + for plugin in plugins: + src_url = plugin.download_url(build, deref=True) + src_ext = os.path.splitext(src_url)[-1] + + try: + sha = prefetch(plugin, build, src_url) + except IOError as e: + print(e, file=sys.stderr) + continue + + build_inputs = [] + if src_ext == ".zip": + build_inputs.append("unzip") + + # TODO: Dependencies are provided as package IDs but refer to both + # internal and external plugins; need to find some way to resolve them + requires = [] + # TODO: Licenses are actually on the website, but aren't provided in the API + license = "lib.licenses.free" + + call_args = [str(builder), "fetchurl", "lib"] + for binput in build_inputs: + call_args.append(binput) + + outfile.write( + f""" + {plugin.packagename()} = callPackage ({{ {", ".join(sorted(call_args))} }}: {builder} {{ + pname = "{plugin.slug}"; + plugname = "{plugin.name}"; + plugid = "{plugin.id}"; + version = "{plugin.version}"; + src = fetchurl {{ + url = "{src_url}"; + sha256 = "{sha}"; + name = "{plugin.filename()}{src_ext}"; + }}; + buildInputs = [ {" ".join(build_inputs)} ]; + packageRequires = [ {" ".join(requires)} ]; + meta = {{ + homepage = "{plugin.url or ""}"; + license = {license}; + description = '' + {plugin.description()} + ''; + }}; + }}) {{}}; +""" + ) + outfile.write("}\n") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "-n", "--number", type=int, help="Limit the number of packages to write", + ) + parser.add_argument( + "-o", "--out", type=str, help="File to write plugins to", + ) + parser.add_argument( + "package", + metavar="PACKAGE", + type=str, + help="The Nixpkgs package name (inc. version)", + ) + + args = parser.parse_args() + + build = Build(args.package) + plugins = list_plugins(build) + plugins.sort(key=lambda p: p.slug) + deduplicate(plugins) + + if args.number: + plugins = plugins[: args.number] + + print(f"Generating packages for {len(plugins)} plugins", file=sys.stderr) + if not args.out: + write_packages(sys.stdout, plugins, build) + else: + with open(args.out, "w") as f: + write_packages(f, plugins, build) + + +main()