basic Python porting to Mojo

[rickysarraf]

'/home/rrs/.modular/pkg/packages.modular.com_max/bin/mojo' '/home/rrs/rrs-home/devel/apt-offline/apt-offline/AptOfflineCoreLib.mojo'
/__w/modular/modular/Kernels/mojo/stdlib/builtin/string_literal.mojo:135:45: error: failed to legalize operation 'pop.string.concat' that was explicitly marked illegal
/__w/modular/modular/Kernels/mojo/stdlib/builtin/string_literal.mojo:135:45: note: see current operation: %2068 = "pop.string.concat"(%201, %2067) : (!kgen.string, !kgen.string) -> !kgen.string
/home/rrs/.modular/pkg/packages.modular.com_max/bin/mojo: error: Failed to materialize symbols: { (exec, { main, KGEN_EE_JIT_GlobalDestructor, KGEN_EE_JIT_GlobalConstructor }) } (from the layer: failed to lower module to LLVM IR for archive compilation)

I got out of ideas when I got this error.

I'm simply trying to take my python code (just the argument/option handling part of it) and port to Mojo, which is when I got the above error.

@ modular host-info
  Host Information
  ================

  Target Triple: x86_64-unknown-linux
  CPU: skylake
  CPU Features: adx, aes, avx, avx2, bmi, bmi2, clflushopt, cmov, crc32, cx16, cx8, f16c, fma, fsgsbase, fxsr, invpcid, lzcnt, mmx, movbe, pclmul, popcnt, prfchw, rdrnd, rdseed, sahf, sgx, sse, sse2, sse3, sse4.1, sse4.2, ssse3, x87, xsave, xsavec, xsaveopt, xsaves

and

@ mojo --version
mojo 24.4.0 (59977802)

[soraros]

Could you maybe share your code?

[rickysarraf]

Yes. Sure.

import os
import sys

from python import Python

var app_name = "apt-offline"
var version = "1.8.5"
var myCopyright = "(C) 2005 - 2022 Ritesh Raj Sarraf"
var terminal_license = "This program comes with ABSOLUTELY NO WARRANTY.\n\
This is free software, and you are welcome to redistribute it under\n\
the GNU GPL Version 3 (or later) License\n"

var errlist = []
var supported_platforms = ["Linux", "GNU/kFreeBSD", "GNU"]
var apt_update_target_path = "/var/lib/apt/lists/partial"
var apt_update_final_path = "/var/lib/apt/lists/"
var apt_package_target_path = "/var/cache/apt/archives/partial/"
var apt_package_final_path = "/var/cache/apt/archives/"

# Locks
var apt_lists_lock = "/var/lib/apt/lists/lock"
var apt_packages_lock = "/var/cache/apt/archives/lock"

var apt_bug_file_format = "__apt__bug__report"

var Bool_Verbose = False

fn main() raises:
    var Py_argparse = Python.import_module("argparse")

    Python.add_to_path(".")
    var Test = Python.import_module("test")

    # Global options
    var global_options = Py_argparse.ArgumentParser(add_help=False)

    global_options.add_argument(
        "--verbose", dest="verbose", help="Enable verbose messages", action="store_true"
    )
    global_options.add_argument(
        "--quiet", dest="quiet", help="Enable quiet mode", action="store_true"
    )

    var parser = Py_argparse.ArgumentParser(prog=app_name, description="Offline APT Package Manager" + " - " + version, epilog=myCopyright + " - " + terminal_license)
    parser.add_argument("-v", "--version", action="version", version=version)

    # We need subparsers for set/get/install
    var subparsers = parser.add_subparsers()

    var Py_builtins = Python.import_module("builtins")

    var setter = Test.Setter()

    # SET command options
    #
    var parser_set = subparsers.add_parser("set")
    parser_set.set_defaults(func=setter.setter)

    parser_set.add_argument(
        "set",
        help="Generate a signature file",
        action="store",
        type=Py_builtins.str,
        metavar="apt-offline.sig",
        default="apt-offline.sig",
    )

    parser_set.add_argument(
        "--simulate",
        dest="set_simulate",
        help="Just simulate. Very helpful when debugging",
        action="store_true",
    )

    # TODO: Handle nargs here.
    parser_set.add_argument(
        "--install-packages",
        dest="set_install_packages",
        help="Packages that need to be installed",
        action="store",
        type=Py_builtins.str,
        nargs="*",
        metavar="PKG",
    )

    parser_set.add_argument(
        "--install-src-packages",
        dest="set_install_src_packages",
        help="Source Packages that need to be installed",
        action="store",
        type=Py_builtins.str,
        nargs="*",
        metavar="SOURCE PKG",
    )

    parser_set.add_argument(
        "--src-build-dep",
        dest="src_build_dep",
        help="Install Build Dependency packages for requested source packages",
        action="store_true",
    )

    parser_set.add_argument(
        "--release",
        dest="set_install_release",
        help="Release target to install packages from",
        action="store",
        type=Py_builtins.str,
        metavar="release_name",
    )

    parser_set.add_argument(
        "--update",
        dest="set_update",
        help="Generate Signature to update APT Database",
        action="store_true",
    )

    parser_set.add_argument(
        "--upgrade",
        dest="set_upgrade",
        help="Generate Signature of packages to be upgraded",
        action="store_true",
    )

    parser_set.add_argument(
        "--upgrade-type",
        dest="upgrade_type",
        help="Type of upgrade to do. Use one of upgrade, dist-upgrade, dselect-ugprade",
        action="store",
        type=Py_builtins.str,
        metavar="upgrade",
        default="upgrade",
    )

    parser_set.add_argument(
        "--generate-changelog",
        dest="generate_changelog",
        help="Generate changelog of the version to be downloaded",
        action="store_true",
    )

    parser_set.add_argument(
        "--apt-backend",
        dest="apt_backend",
        help="APT backend to use. One of: apt, apt-get, python-apt",
        action="store",
        type=Py_builtins.str,
        metavar="apt-get",
        default="apt-get",
    )

    # GET command options
    var parser_get = subparsers.add_parser("get")

    var fetcher = Test.Fetcher()

    # INFO: When get option is called, call the fetcher() function
    parser_get.set_defaults(func=fetcher.fetcher)

    parser_get.add_argument(
        "get",
        help="Get apt-offline data",
        action="store",
        type=Py_builtins.str,
        metavar="apt-offline.sig",
        default="apt-offline.sig",
    )

    parser_get.add_argument(
        "--socket-timeout",
        dest="socket_timeout",
        help="Set Socket Timeout",
        action="store",
        type=Py_builtins.int,
        metavar="30",
        default=30,
    )

    parser_get.add_argument(
        "-d",
        "--download-dir",
        dest="download_dir",
        help="Folder path to save files to",
        action="store",
        type=Py_builtins.str,
        metavar="apt-downloads",
    )

    parser_get.add_argument(
        "-s",
        "--cache-dir",
        dest="cache_dir",
        help="Cache folder to search for",
        action="store",
        type=Py_builtins.str,
        metavar="/var/cache/apt/archives/",
    )

    parser_get.add_argument(
        "--no-checksum",
        dest="disable_md5check",
        help="Do not validate checksum of downloaded files",
        action="store_true",
    )

    parser_get.add_argument(
        "-t",
        "--threads",
        dest="num_of_threads",
        help="Number of threads to spawn",
        action="store",
        type=Py_builtins.int,
        metavar="1",
        default=1,
    )

    parser_get.add_argument(
        "--bundle",
        dest="bundle_file",
        help="Bundle output data to a file",
        action="store",
        type=Py_builtins.str,
        metavar="apt-offline-bundle.zip",
    )

    parser_get.add_argument(
        "--bug-reports",
        dest="deb_bugs",
        help="Fetch bug reports from the BTS",
        action="store_true",
    )

    parser_get.add_argument(
        "--proxy-host",
        dest="proxy_host",
        help="Proxy Host to use",
        type=Py_builtins.str,
        default=None,
    )

    parser_get.add_argument(
        "--proxy-port",
        dest="proxy_port",
        help="Proxy port number to use",
        type=Py_builtins.int,
        default=None,
    )

    parser_get.add_argument(
        "--https-cert-file",
        dest="https_cert_file",
        help="Certificate file for https client authentication",
        type=Py_builtins.str,
        default=None,
    )

    parser_get.add_argument(
        "--https-key-file",
        dest="https_key_file",
        help="Certificate key for https client authentication",
        type=Py_builtins.str,
        default=None,
    )

    parser_get.add_argument(
        "--http-basicauth",
        dest="http_basicauth",
        action="append",
        help="A user/password encoded URL. Passwords with special characters should be percent encoded",
        type=Py_builtins.str,
        default=[],
        metavar="https://username:[email protected]/",
    )

    parser_get.add_argument(
        "--disable-cert-check",
        dest="disable_cert_check",
        help="Disable Certificate check on https connections",
        action="store_true",
    )

    # INSTALL command options
    var parser_install = subparsers.add_parser("install")

    var installer = Test.Installer()

    parser_install.set_defaults(func=installer.installer)

    parser_install.add_argument(
        "install",
        help="Install apt-offline data, a bundle file or a directory",
        action="store",
        type=Py_builtins.str,
        metavar="apt-offline-download.zip | apt-offline-download/",
    )

    parser_install.add_argument(
        "--simulate",
        dest="install_simulate",
        help="Just simulate. Very helpful when debugging",
        action="store_true",
    )

    parser_install.add_argument(
        "--install-src-path",
        dest="install_src_path",
        help="Install src packages to specified path.",
        default=None,
    )

    parser_install.add_argument(
        "--extra-keyring",
        dest="extra_keyring",
        help="Extra Keyring path to include for.",
        metavar="/etc/apt/keyring/",
        default=None,
    )

    parser_install.add_argument(
        "--skip-bug-reports",
        dest="skip_bug_reports",
        help="Skip the bug report check",
        action="store_true",
    )

    parser_install.add_argument(
        "--skip-changelog",
        dest="skip_changelog",
        help="Skip display of changelog",
        action="store_true",
    )

    parser_install.add_argument(
        "--allow-unauthenticated",
        dest="allow_unauthenticated",
        help="Ignore apt gpg signatures mismatch",
        action="store_true",
    )

    parser_install.add_argument(
        "--strict-deb-check",
        dest="strict_deb_check",
        help="Perform strict checksum validaton for downloaded .deb files",
        action="store_true",
    )

    var args = parser.parse_args()

    print(app_name)

and to overcome the current limitations of Python <=> Mojo:

test.py:

class Setter:
    def setter(self):
        pass

class Fetcher:
    def fetcher(self):
        pass

class Installer:
    def installer(self):
        pass

References

• How to convert ArgumentParser code to Mojo? #1029
• [Feature Request]: Allow passing Mojo functions as callbacks to Python function #702

[soraros]

In Mojo, a string literal ("something") has the type StringLiteral (this is about to change to match Python). They can't be concatenated at compile time. The fix is simple: convert them to String as follows:

...
    var parser = Py_argparse.ArgumentParser(
        prog=app_name,
        description=String("Offline APT Package Manager") + " - " + version,
        epilog=String(myCopyright) + " - " + terminal_license,
    )
...

Perhaps the error message is less helpful on version 24.4 (I don't have it installed), but the error on the nightly build reads:

/__w/modular/modular/open-source/mojo/stdlib/stdlib/builtin/string_literal.mojo:93:45: error: failed to legalize operation 'pop.string.concat' that was explicitly marked illegal
/workspaces/nightly/mojo-dev/stdlib/main.mojo:44:110: note: called from
    var parser = Py_argparse.ArgumentParser(prog=app_name, description="Offline APT Package Manager" + " - " + version, epilog=myCopyright + " - " + terminal_license)

This error points out the exact problem, and where it occurs.

[rickysarraf]

Thank you. THat indeed was the cause. 🙏🏽