(ns libpython-clj2.python

"Python bindings for Clojure. This library dynamically finds the installed

(:require [libpython-clj2.python.info :as py-info]

[libpython-clj2.python.ffi :as py-ffi]

[libpython-clj2.python.base :as py-base]

[libpython-clj2.python.fn :as py-fn]

[libpython-clj2.python.protocols :as py-proto]

[libpython-clj2.python.class :as py-class]

[libpython-clj2.python.with :as py-with]

[libpython-clj2.python.dechunk-map :refer [dechunk-map]]

[libpython-clj2.python.copy :as py-copy]

[libpython-clj2.python.bridge-as-jvm :as py-bridge-jvm]

[libpython-clj2.python.bridge-as-python]

[libpython-clj2.python.io-redirect :as io-redirect]

[libpython-clj2.python.gc :as pygc]

[libpython-clj2.python.windows :as win]

[tech.v3.datatype.ffi :as dtype-ffi]

[tech.v3.datatype.errors :as errors]

[clojure.tools.logging :as log]

clojure.edn)

(:import [java.util Map List]

[clojure.lang IFn]))

(set! *warn-on-reflection* true)

(defn initialize!

"Initialize the python library. If library path is not provided, then the system

[& {:keys [windows-anaconda-activate-bat

library-path

no-io-redirect?]

:as options}]

(if-not (and (py-ffi/library-loaded?)

(= 1 (py-ffi/Py_IsInitialized)))

(let [python-edn-opts (-> (try (slurp "python.edn")

(catch java.io.FileNotFoundException _ "{}"))

clojure.edn/read-string)

_ (some-> python-edn-opts :pre-initialize-fn requiring-resolve (apply []))

options (merge python-edn-opts options)

info (py-info/detect-startup-info options)

_ (log/infof "Startup info %s" info)

_ (when-let [lib-path (:java-library-path-addendum

options (:java-library-path-addendum info))]

(log/infof "Prefixing java library path: %s" lib-path)

(py-ffi/append-java-library-path! lib-path))

libname (->> (concat (when library-path [library-path]) (:libnames info))

(dechunk-map identity)

(map dtype-ffi/find-library)

(remove nil?)

(first))]

(errors/when-not-errorf

libname

"Failed to find a valid python library!")

(log/infof "Loading python library: %s" libname)

(py-ffi/initialize!

libname (:python-home info)

(assoc options

:program-name (:program-name options (:executable info))

:python-home (:python-home options (:python-home info))

:java-library-path-addendum (:java-library-path-addendum

options

(:java-library-path-addendum info))))

(let [gilstate (py-ffi/lock-gil)]

(try

(when-not (nil? windows-anaconda-activate-bat)

(win/setup-windows-conda! windows-anaconda-activate-bat

py-ffi/run-simple-string))

(when-not no-io-redirect?

(io-redirect/redirect-io!))

(finally

(py-ffi/unlock-gil gilstate))))

(some-> python-edn-opts :post-initialize-fn requiring-resolve (apply []))

:ok)

:already-initialized))

(defmacro stack-resource-context

"Create a stack-based resource context. All python objects allocated within this

[& body]

`(pygc/with-stack-context

~@body))

(defmacro with-gil

"Capture the gil for an extended amount of time. This can greatly speed up

[& body]

`(py-ffi/with-gil

~@body))

(defmacro with-gil-stack-rc-context

"Capture the gil, open a resource context. The resource context is released

[& body]

`(py-ffi/with-gil

(pygc/with-stack-context

~@body)))

(defmacro with-manual-gil

"When running with -Dlibpython_clj.manual_gil=true, you need to wrap all accesses to

[& body]

`(with-open [locker# (py-ffi/manual-gil-locker)]

~@body))

(defmacro with-manual-gil-stack-rc-context

"When running with -Dlibpython_clj.manual_gil=true, you need to wrap all accesses to

[& body]

`(with-manual-gil

(pygc/with-stack-context

~@body)))

(declare ->jvm)

(defn ^:no-doc in-py-ctx

[^java.util.function.Supplier supplier]

(with-gil-stack-rc-context

(-> (.get supplier)

(->jvm))))

(defn import-module

"Import a python module. Module entries can be accessed via get-attr."

[modname]

(with-gil

(if-let [mod (py-ffi/PyImport_ImportModule modname)]

(-> (py-ffi/track-pyobject mod)

(py-base/as-jvm))

(py-ffi/check-error-throw))))

(defn add-module

"Add a python module. This can create a module if it doesn't exist."

[modname]

(with-gil

(-> (py-ffi/PyImport_AddModule modname)

(py-ffi/incref-track-pyobject)

(py-base/as-jvm))))

(defn module-dict

"Get the module dictionary."

[mod]

(with-gil

(-> (py-ffi/PyModule_GetDict mod)

(py-ffi/incref-track-pyobject)

(py-base/as-jvm))))

(defn dir

[pyobj]

(with-gil (py-proto/dir pyobj)))

(defn call-attr

"Call an attribute on a python object using only positional arguments"

[pyobj attname & args]

(with-gil (py-fn/call-attr pyobj attname args)))

(defn call-attr-kw

"Call an attribute passing in both positional and keyword arguments."

[pyobj attname args kw-list]

(with-gil (py-fn/call-attr-kw pyobj attname args kw-list py-base/as-python)))

(defn get-attr

"Get an attribute from a python object"

[pyobj attname]

(with-gil (py-proto/get-attr pyobj attname)))

(defn set-attr!

"Set an attribute on a python object. Returns pyobj."

[pyobj attname attval]

(with-gil (py-proto/set-attr! pyobj attname attval))

pyobj)

(defn set-attrs!

"Set a sequence of [name value] attributes. Returns pyobj."

[pyobj att-seq]

(with-gil (doseq [[k v] att-seq] (set-attr! pyobj k v)))

pyobj)

(defn has-attr?

"Return true if this python object has this attribute."

[pyobj att-name]

(py-proto/has-attr? pyobj att-name))

(defn get-item

"Get an item from a python object using __getitem__"

[pyobj item-name]

(with-gil (py-proto/get-item pyobj item-name)))

(defn set-item!

"Set an item on a python object using __setitem__"

[pyobj item-name item-val]

(with-gil (py-proto/set-item! pyobj item-name item-val))

pyobj)

(defn has-item?

"Return true if the python object has an item. Calls __hasitem__."

[pyobj item-name]

(with-gil (py-proto/has-item? pyobj item-name)))

(defn set-items!

"Set a sequence of [name value]. Returns pyobj"

[pyobj item-seq]

(with-gil (doseq [[k v] item-seq] (set-item! pyobj k v)))

pyobj)

(defn ->python

"Copy a jvm value into a python object"

[v]

(py-ffi/with-gil (py-base/->python v)))

(defn as-python

"Bridge a jvm value into a python object"

[v]

(py-ffi/with-gil (py-base/as-python v)))

(defn ->jvm

"Copy a python value into java datastructures"

[v & [opts]]

(py-ffi/with-gil (py-base/->jvm v opts)))

(defn as-jvm

"Copy a python value into java datastructures"

[v & [opts]]

(py-ffi/with-gil (py-base/as-jvm v opts)))

(defn as-map

"Make a python object appear as a map of it's items"

^Map [pobj]

(py-bridge-jvm/generic-python-as-map (delay pobj)))

(defn as-list

"Make a python object appear as a list"

^List [pobj]

(py-bridge-jvm/generic-python-as-list (delay pobj)))

(defn python-type

"Get the type (as a keyword) of a python object"

[v]

(if v

(py-ffi/with-gil (py-proto/python-type v))

:py-none))

(defn attr-type-map

"Return a map of attr name to python-type of the attribute"

[pyobj]

(py-ffi/with-gil

(into (sorted-map) (map #(vector % (python-type (get-attr pyobj %))))

(dir pyobj))))

(defmacro import-as

"Import a module and assign it to a var. Documentation is included."

[module-path varname]

`(let [~'mod-data (import-module ~(name module-path))]

(def ~varname (import-module ~(name module-path)))

(alter-meta! #'~varname assoc :doc (get-attr ~'mod-data "__doc__"))

#'~varname))

(defn ^:no-doc py-var-metadata [var-name var-data]

(try

(let [metadata-fn (requiring-resolve 'libpython-clj2.metadata/py-fn-metadata)]

(select-keys (metadata-fn var-name var-data {}) [:doc :arglists]))

(catch Throwable _

{:doc (get-attr var-data "__doc__")})))

(defmacro from-import

"Support for the from a import b,c style of importing modules and symbols in python.

[module-path item & args]

`(do

(let [~'mod-data (import-module ~(name module-path))]

~@(map (fn [varname]

`(let [~'var-data (get-attr ~'mod-data ~(name varname))]

(def ~varname ~'var-data)

(alter-meta! #'~varname merge (py-var-metadata ~(name varname) ~'var-data))

#'~varname))

(concat [item] args)))))

(defn is-instance?

"Return true if inst is an instance of cls. Note that arguments

[py-inst py-cls]

(py-ffi/with-gil

(let [retval (long (py-ffi/PyObject_IsInstance py-inst py-cls))]

(case retval

0 false

1 true

(py-ffi/check-error-throw)))))

(defn callable?

"Return true if python object is callable."

[pyobj]

(cond

(instance? IFn pyobj)

true

(py-ffi/convertible-to-pointer? pyobj)

(py-ffi/with-gil

(let [retval (long (py-ffi/PyCallable_Check pyobj))]

(case retval

0 false

1 true

(py-ffi/check-error-throw))))

:else

false))

(defn ->py-list

"Copy the data into a python list"

[v]

(py-ffi/with-gil (-> (py-copy/->py-list v) (as-jvm))))

(defn ->py-tuple

"Copy v into a python tuple"

[v]

(py-ffi/with-gil (-> (py-copy/->py-tuple v) (as-jvm))))

(defn ->py-dict

"Copy v into a python dict"

[v]

(py-ffi/with-gil (-> (py-copy/->py-dict v) (as-jvm))))

(defn run-simple-string

"Run a string expression returning a map of

[program & {:keys [globals locals]}]

(->> (py-ffi/run-simple-string program :globals globals :locals locals)

(map (fn [[k v]]

[k (py-base/as-jvm v)]))

(into {})))

(defn make-callable

"Make a python callable object from a clojure function. This is called for you

([ifn options]

(py-fn/make-tuple-fn ifn options))

([ifn] (make-callable ifn nil)))

(defn ^:no-doc make-tuple-fn

"Deprecated - use make-callable"

[ifn & {:as options}]

(make-callable ifn options))

(defn make-instance-fn

"Make an callable instance function - a function which will be passed the 'this'

([ifn options] (py-class/make-tuple-instance-fn ifn options))

([ifn] (make-instance-fn ifn nil)))

(defn make-kw-instance-fn

"Make an kw callable instance function - function by default is passed 2 arguments,

([ifn options] (py-class/make-kw-instance-fn ifn options))

([ifn] (make-kw-instance-fn ifn nil)))

(defn ^:no-doc make-tuple-instance-fn

[ifn & {:as options}]

(make-instance-fn ifn options))

(defn create-class

"Create a new class object. Any callable values in the cls-hashmap

[name bases cls-hashmap]

(py-class/create-class name bases cls-hashmap))

(defn cfn

"Call an object.

[item & args]

(apply py-fn/cfn item args))

(defn afn

"Call an attribute of an object.

[item attr & args]

(apply py-fn/afn item attr args))

(defn make-fastcallable

"Wrap a python callable such that calling it in a tight loop with purely positional

^java.lang.AutoCloseable [item]

(py-fn/make-fastcallable item))

(defmacro with

"Support for the 'with' statement in python:

[bind-vec & body]

`(py-with/with ~bind-vec ~@body))

(defmacro $a

"Call an attribute of an object using automatic detection of the python kwargs.

[item attr & args]

(let [[pos-args kw-args] (py-fn/args->pos-kw-args args)]

`(call-attr-kw ~item ~(py-fn/key-sym-str->str attr)

~pos-args ~kw-args)))

(defmacro $c

"Call an object using automatic detection of the python kwargs.

[item & args]

(let [[pos-args kw-args] (py-fn/args->pos-kw-args args)]

`(py-fn/call-kw ~item ~pos-args ~kw-args)))

(defmacro py.-

"Class/object getter syntax. (py.- obj attr) is equivalent to

[x arg]

`(get-attr ~x ~(py-fn/key-sym-str->str arg)))

(defmacro py.

"Class/object method syntax. (py. obj method arg1 arg2 ... argN)

[x method-name & args]

;; method-name cast to a string specifically for go and go-loop

;; compatability

`(~#'$a ~x ~(str method-name) ~@args))

(defmacro py*

"Special syntax for passing along *args and **kwargs style arguments

([x method args]

(list #'call-attr-kw x (py-fn/key-sym-str->str method) args nil))

([x method args kwargs]

(list #'call-attr-kw x (py-fn/key-sym-str->str method) args kwargs)))

(defmacro py**

"Like py*, but it is assumed that the LAST argument is kwargs."

([x method kwargs]

(list #'call-attr-kw x (str method) nil kwargs))

([x method arg & args]

(let [args (into [arg] args)

kwargs (last args)

args (vec (pop args))]

(list #'call-attr-kw x (py-fn/key-sym-str->str method) args kwargs))))

(defn ^:private handle-pydotdot

([x form]

(if (list? form)

(let [form-data (vec form)

[instance-member & args] form-data

symbol-str (str instance-member)]

(cond

(clojure.string/starts-with? symbol-str "-")

(list #'py.- x (symbol (subs symbol-str 1 (count symbol-str))))

(clojure.string/starts-with? symbol-str "**")

(list* #'py** x (symbol (subs symbol-str 2 (count symbol-str))) args)

(clojure.string/starts-with? symbol-str "*")

(list* #'py* x (symbol (subs symbol-str 1 (count symbol-str))) args)

:else ;; assumed to be method invocation

(list* (into (vector #'py. x instance-member) args))))

(handle-pydotdot x (list form))))

([x form & more]

(apply handle-pydotdot (handle-pydotdot x form) more)))

(defmacro py..

"Extended accessor notation, similar to the `..` macro in Clojure.

[x & args]

(apply handle-pydotdot x args))

(defn- module-path-string

"Given a.b, return a

[x]

(clojure.string/join

"."

(pop (clojure.string/split (str x) #"[.]"))))

(defn- module-path-last-string

"Given a.b.c.d, return d"

[x]

(last (clojure.string/split (str x) #"[.]")))

(defn path->py-obj

"Given a string such as \"builtins\" or \"builtins.list\", load the module or

[item-path & {:keys [reload?]}]

(when (seq item-path)

(if-let [module-retval (try

(import-module item-path)

(catch Exception e

(when-not (seq (module-path-string item-path))

(throw e))))]

(if reload?

(let [import-lib (import-module "importlib")]

(call-attr import-lib "reload" module-retval))

module-retval)

(let [butlast (module-path-string item-path)]

(if-let [parent-mod (path->py-obj butlast :reload? reload?)]

(get-attr parent-mod (module-path-last-string item-path))

(throw (Exception. (format "Failed to find module or class %s"

item-path))))))))

(defmacro def-unpack

"Unpack a set of symbols into a set of defs. Useful when trying to match Python

[symbols input]

`(let [~symbols ~input]

~@(for [s symbols] `(def ~s ~s))))