New flow CLI, SDK

.github/workflows/test.yml

@@ -37,8 +37,11 @@ jobs:
 
     - name: Install Python ${{ matrix.ver }} dependencies
       run: |
-        python3 -m pip install --upgrade pip setuptools
-        python3 -m pip install tox numpy
+        python3 -m pip install --upgrade pip setuptools wheel
+        python3 -m pip install -e .[test]
+
+    - name: Execute pytest tests
+      run: pytest
 
     - name: Execute Python tests
       run: tox
@@ -75,5 +78,3 @@ jobs:
       run: |
         cd R/tests
         Rscript run_tests.R
-
-

MANIFEST.in

@@ -1,4 +1,6 @@
 include LICENSE
 include metaflow/plugins/cards/card_modules/*.html
 include metaflow/plugins/cards/card_modules/*.js
-include metaflow/plugins/cards/card_modules/*.css
+include metaflow/plugins/cards/card_modules/*.css
+include requirements.txt
+include requirements-test.txt

README.md

@@ -1,5 +1,8 @@
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)
 
+[celsiustx/metaflow fork, `dsl` branch](https://github.com/celsiustx/metaflow/tree/dsl/metaflow/api): see [`metaflow/api`](metaflow/api)
+
+-------
 # Metaflow
 
 Metaflow is a human-friendly Python/R library that helps scientists and engineers build and manage real-life data science projects. Metaflow was originally developed at Netflix to boost productivity of data scientists who work on a wide variety of projects from classical statistics to state-of-the-art deep learning.

conftest.py

@@ -0,0 +1 @@
+collect_ignore = ["metaflow/plugins", "test/core", "test/data", "test/unit"]

metaflow/R.py

@@ -2,6 +2,8 @@
 import imp
 from tempfile import NamedTemporaryFile
 
+from metaflow.parameters import register_main_flow
+
 from .util import to_bytes
 
 R_FUNCTIONS = {}
@@ -103,6 +105,8 @@ def run(
     module = imp.load_source("metaflowR", tmp.name)
     flow = module.FLOW(use_cli=False)
 
+    register_main_flow(module.FLOW)
+
     from . import exception
     from . import cli
 
@@ -112,6 +116,7 @@ def run(
             args=metaflow_args,
             handle_exceptions=False,
             entrypoint=full_cmdline[: -len(metaflow_args)],
+            standalone_mode=False,
         )
     except exception.MetaflowException as e:
         cli.print_metaflow_exception(e)

metaflow/__init__.py

@@ -101,7 +101,7 @@ class and related decorators.
 from .current import current
 
 # Flow spec
-from .flowspec import FlowSpec
+from .flowspec import FlowSpec, FlowSpecMeta
 
 from .parameters import Parameter, JSONTypeClass
 

metaflow/_vendor/click/_compat.py

@@ -270,7 +270,6 @@ def filename_to_ui(value):
             value = value.decode(get_filesystem_encoding(), "replace")
         return value
 
-
 else:
     import io
 
@@ -725,7 +724,6 @@ def get_winterm_size():
             ).srWindow
             return win.Right - win.Left, win.Bottom - win.Top
 
-
 else:
 
     def _get_argv_encoding():

metaflow/_vendor/click/_termui_impl.py

@@ -459,7 +459,9 @@ def edit_file(self, filename):
             environ = None
         try:
             c = subprocess.Popen(
-                '{} "{}"'.format(editor, filename), env=environ, shell=True,
+                '{} "{}"'.format(editor, filename),
+                env=environ,
+                shell=True,
             )
             exit_code = c.wait()
             if exit_code != 0:
@@ -563,11 +565,11 @@ def _unquote_file(url):
 
 
 def _translate_ch_to_exc(ch):
-    if ch == u"\x03":
+    if ch == "\x03":
         raise KeyboardInterrupt()
-    if ch == u"\x04" and not WIN:  # Unix-like, Ctrl+D
+    if ch == "\x04" and not WIN:  # Unix-like, Ctrl+D
         raise EOFError()
-    if ch == u"\x1a" and WIN:  # Windows, Ctrl+Z
+    if ch == "\x1a" and WIN:  # Windows, Ctrl+Z
         raise EOFError()
 
 
@@ -614,14 +616,13 @@ def getchar(echo):
             func = msvcrt.getwch
 
         rv = func()
-        if rv in (u"\x00", u"\xe0"):
+        if rv in ("\x00", "\xe0"):
             # \x00 and \xe0 are control characters that indicate special key,
             # see above.
             rv += func()
         _translate_ch_to_exc(rv)
         return rv
 
-
 else:
     import tty
     import termios

metaflow/_vendor/click/globals.py

@@ -36,7 +36,7 @@ def pop_context():
 
 
 def resolve_color_default(color=None):
-    """"Internal helper to get the default value of the color flag.  If a
+    """ "Internal helper to get the default value of the color flag.  If a
     value is passed it's returned unchanged, otherwise it's looked up from
     the current context.
     """

metaflow/_vendor/click/utils.py

@@ -234,9 +234,9 @@ def echo(message=None, file=None, nl=True, err=False, color=None):
         message = text_type(message)
 
     if nl:
-        message = message or u""
+        message = message or ""
         if isinstance(message, text_type):
-            message += u"\n"
+            message += "\n"
         else:
             message += b"\n"
 

metaflow/_vendor/v3_5/__init__.py

@@ -1 +1 @@
-# Empty file
+# Empty file