added test to http pipe

bale/content.py

@@ -1,3 +1,4 @@
+import asyncio
 from nicegui import ui
 from bale import elements as el
 import bale.logo as logo
@@ -26,7 +27,7 @@ class Content:
         self._automation = None
         self._history = None
 
-    def build(self):
+    async def build(self):
         self._header = ui.header(bordered=True).classes("bg-dark q-pt-sm q-pb-xs")
         self._header.tailwind.border_color(f"[{el.orange}]").min_width("[920px]")
         self._header.visible = False
@@ -44,11 +45,7 @@ class Content:
         self._tab_panels = (
             ui.tab_panels(self._tabs, value="Manage", on_change=lambda e: self._tab_changed(e), animated=False).classes("w-full h-full").bind_visibility_from(self._header)
         )
-        ui.timer(1, self.select_default, once=True)
-
-    async def select_default(self):
-        tab = Tab(spinner=None)
-        default = tab.common.get("default", "")
+        default = Tab(spinner=None).common.get("default", "")
         if default != "":
             await self.host_selected(default)
 

bale/elements.py

@@ -162,16 +162,17 @@ class FInput(ui.input):
 class DSelect(ui.select):
     def __init__(
         self,
-        options: List | Dict,
+        options: Union[List, Dict],
         *,
-        label: str | None = None,
+        label: Optional[str] = None,
         value: Any = None,
-        on_change: Callable[..., Any] | None = None,
+        on_change: Optional[Callable[..., Any]] = None,
         with_input: bool = False,
+        new_value_mode: Optional[Literal["add", "add-unique", "toggle"]] = None,
         multiple: bool = False,
         clearable: bool = False,
     ) -> None:
-        super().__init__(options, label=label, value=value, on_change=on_change, with_input=with_input, multiple=multiple, clearable=clearable)
+        super().__init__(options, label=label, value=value, on_change=on_change, with_input=with_input, new_value_mode=new_value_mode, multiple=multiple, clearable=clearable)
         self.tailwind.width("full")
         if multiple is True:
             self.props("use-chips")
@@ -180,16 +181,17 @@ class DSelect(ui.select):
 class FSelect(ui.select):
     def __init__(
         self,
-        options: List | Dict,
+        options: Union[List, Dict],
         *,
-        label: str | None = None,
+        label: Optional[str] = None,
         value: Any = None,
-        on_change: Callable[..., Any] | None = None,
+        on_change: Optional[Callable[..., Any]] = None,
         with_input: bool = False,
+        new_value_mode: Optional[Literal["add", "add-unique", "toggle"]] = None,
         multiple: bool = False,
         clearable: bool = False,
     ) -> None:
-        super().__init__(options, label=label, value=value, on_change=on_change, with_input=with_input, multiple=multiple, clearable=clearable)
+        super().__init__(options, label=label, value=value, on_change=on_change, with_input=with_input, new_value_mode=new_value_mode, multiple=multiple, clearable=clearable)
         self.tailwind.width("64")
 
 

bale/interfaces/cli.py

@@ -34,6 +34,7 @@ class Cli:
         self.stderr: List[str] = []
         self._terminate: asyncio.Event = asyncio.Event()
         self._busy: bool = False
+        self._truncated: bool = False
         self.prefix_line: str = ""
         self._stdout_terminals: List[Terminal] = []
         self._stderr_terminals: List[Terminal] = []
@@ -70,8 +71,11 @@ class Cli:
             else:
                 break
 
-    async def _controller(self, process: Process) -> None:
+    async def _controller(self, process: Process, max_output_lines) -> None:
         while process.returncode is None:
+            if max_output_lines > 0 and len(self.stderr) + len(self.stdout) > max_output_lines:
+                self._truncated = True
+                process.terminate()
             if self._terminate.is_set():
                 process.terminate()
             try:
@@ -83,7 +87,7 @@ class Cli:
     def terminate(self) -> None:
         self._terminate.set()
 
-    async def execute(self, command: str) -> Result:
+    async def execute(self, command: str, max_output_lines: int = 0) -> Result:
         self._busy = True
         c = shlex.split(command, posix=False)
         try:
@@ -92,13 +96,14 @@ class Cli:
                 self.stdout.clear()
                 self.stderr.clear()
                 self._terminate.clear()
+                self._truncated = False
                 terminated = False
                 now = datetime.now().strftime("%Y/%m/%d %H:%M:%S")
                 self.prefix_line = f"<{now}> {command}\n"
                 for terminal in self._stdout_terminals:
                     terminal.call_terminal_method("write", "\n" + self.prefix_line)
                 await asyncio.gather(
-                    self._controller(process=process),
+                    self._controller(process=process, max_output_lines=max_output_lines),
                     self._read_stdout(stream=process.stdout),
                     self._read_stderr(stream=process.stderr),
                 )
@@ -110,7 +115,9 @@ class Cli:
         finally:
             self._terminate.clear()
             self._busy = False
-        return Result(command=command, return_code=process.returncode, stdout_lines=self.stdout.copy(), stderr_lines=self.stderr.copy(), terminated=terminated)
+        return Result(
+            command=command, return_code=process.returncode, stdout_lines=self.stdout.copy(), stderr_lines=self.stderr.copy(), terminated=terminated, truncated=self._truncated
+        )
 
     async def shell(self, command: str) -> Result:
         self._busy = True

bale/interfaces/ssh.py

@@ -90,10 +90,10 @@ class Ssh(Cli):
         del self._config[self.host]
         self.write_config()
 
-    async def execute(self, command: str) -> Result:
+    async def execute(self, command: str, max_output_lines: int = 0) -> Result:
         self._base_cmd = f"{'' if self.use_key else f'sshpass -p {self.password} '} ssh -F {self._config_path} {self.host}"
         self._full_cmd = f"{self._base_cmd} {command}"
-        return await super().execute(self._full_cmd)
+        return await super().execute(self._full_cmd, max_output_lines)
 
     async def send_key(self) -> Result:
         await get_public_key(self._raw_path)

bale/interfaces/sshdl.py

@@ -3,7 +3,7 @@ from pathlib import Path
 import stat
 from datetime import datetime
 import uuid
-from nicegui import app, events, ui
+from nicegui import app, background_tasks, events, ui
 from fastapi.responses import StreamingResponse
 import asyncssh
 from bale import elements as el
@@ -87,7 +87,7 @@ class SshFileBrowse(ui.dialog):
                     row.tailwind.height("[40px]")
                     el.DButton("Download", on_click=self._start_download)
                     ui.button("Exit", on_click=lambda: self.submit("exit"))
-        await self._update_grid()
+        await self._update_handler()
 
     async def _connect(self) -> Tuple[asyncssh.SSHClientConnection, asyncssh.SFTPClient]:
         ssh = await asyncssh.connect(self._zfs.hostname, username=self._zfs.username, client_keys=[self._zfs.key_path])
@@ -137,7 +137,7 @@ class SshFileBrowse(ui.dialog):
             "permissions": attributes.permissions,
         }
 
-    async def _update_grid(self) -> None:
+    async def _update_handler(self) -> None:
         self._grid.call_api_method("showLoadingOverlay")
         if self._ssh is None or self._sftp is None:
             self._ssh, self._sftp = await self._connect()
@@ -165,7 +165,7 @@ class SshFileBrowse(ui.dialog):
     async def _handle_double_click(self, e: events.GenericEventArguments) -> None:
         self.path = e.args["data"]["path"]
         if e.args["data"]["type"] == "directory":
-            await self._update_grid()
+            await self._update_handler()
         else:
             await self._start_download(e)
 
@@ -226,10 +226,10 @@ class SshFileFind(SshFileBrowse):
             with el.DBody(height="fit", width="[90vw]"):
                 with el.WColumn().classes("col"):
                     filesystems = await self._zfs.filesystems
-                    self._filesystem = el.DSelect(
-                        list(filesystems.data.keys()), label="filesystem", with_input=True, on_change=self._update_grid
-                    )
-                    self._pattern = el.DInput("Pattern", on_change=self._update_grid)
+                    self._filesystem = el.DSelect(list(filesystems.data.keys()), label="filesystem", with_input=True, on_change=self._update_handler)
+                    with el.WRow():
+                        self._pattern = ui.input("Pattern").classes("col").on("keydown.enter", handler=self._update_handler)
+                        el.LgButton(icon="search", on_click=self._update_handler)
                     self._grid = ui.aggrid(
                         {
                             "defaultColDef": {"flex": 1, "sortable": True, "suppressMovable": True, "sortingOrder": ["asc", "desc"]},
@@ -237,12 +237,14 @@ class SshFileFind(SshFileBrowse):
                                 {"field": "name", "headerName": "Name", "flex": 1, "sort": "desc", "resizable": True},
                                 {"field": "location", "headerName": "Location", "flex": 1, "resizable": True},
                                 {
-                                    "field": "modified_datetime",
                                     "headerName": "Modified",
+                                    "field": "modified_timestamp",
+                                    "filter": "agTextColumnFilter",
                                     "maxWidth": 200,
-                                    ":comparator": """(valueA, valueB, nodeA, nodeB, isInverted) => {
-                                            return (nodeA.data.modified_timestamp > nodeB.data.modified_timestamp) ? -1 : 1;
-                                        }""",
+                                    ":cellRenderer": """(data) => {
+                                        var date = new Date(data.value * 1000).toLocaleString(undefined, {dateStyle: 'short', timeStyle: 'short', hour12: false});;
+                                        return date;
+                                    }""",
                                 },
                                 {
                                     "field": "size",
@@ -264,15 +266,21 @@ class SshFileFind(SshFileBrowse):
                     row.tailwind.height("[40px]")
                     el.DButton("Download", on_click=self._start_download)
                     ui.button("Exit", on_click=lambda: self.submit("exit"))
-            await self._update_grid()
+                self._grid.call_api_method("hideOverlay")
 
-    async def _update_grid(self) -> None:
-        self._grid.call_api_method("showLoadingOverlay")
-        if self._filesystem is not None:
+    async def _update_handler(self) -> None:
+        if len(self._pattern.value) > 0 and self._filesystem is not None:
+            self._grid.call_api_method("showLoadingOverlay")
+            self._filesystem.props("readonly")
+            self._pattern.props("readonly")
             files = await self._zfs.find_files_in_snapshots(filesystem=self._filesystem.value, pattern=self._pattern.value)
             self._grid.options["rowData"] = files.data
-        self._grid.update()
-        self._grid.call_api_method("hideOverlay")
+            if files.truncated is True:
+                el.notify("Too many files found, truncating list.", type="warning")
+            self._grid.update()
+            self._filesystem.props(remove="readonly")
+            self._pattern.props(remove="readonly")
+            self._grid.call_api_method("hideOverlay")
 
     async def _handle_double_click(self, e: events.GenericEventArguments) -> None:
         await self._start_download(e)

bale/interfaces/zfs.py

@@ -81,7 +81,7 @@ class Zfs:
         command = command if len(command) < 160 else command[:160] + "..."
         el.notify(command)
 
-    async def execute(self, command: str, notify: bool = True) -> Result:
+    async def execute(self, command: str, max_output_lines: int = 0, notify: bool = True) -> Result:
         if notify:
             self.notify(command)
         return Result(command=command)
@@ -166,25 +166,28 @@ class Zfs:
         return result
 
     async def find_files_in_snapshots(self, filesystem: str, pattern: str) -> Result:
-        filesystems = await self.filesystems
-        if filesystem in filesystems.data.keys():
-            if "mountpoint" in filesystems.data[filesystem]:
-                command = f"find {filesystems.data[filesystem]['mountpoint']}/.zfs/snapshot -type f -name '{pattern}' -printf '%h\t%f\t%s\t%T@\n'"
-                result = await self.execute(command=command, notify=False)
-                files = []
-                for line in result.stdout_lines:
-                    matches = re.match(
-                        "^(?P<location>[^\t]+)\t(?P<name>[^\t]+)\t(?P<bytes>[^\t]+)\t(?P<modified_timestamp>[^\n]+)",
-                        line,
-                    )
-                    if matches is not None:
-                        md = matches.groupdict()
-                        md["path"] = f"{md['location']}/{md['name']}"
-                        md["size"] = format_bytes(int(md["bytes"]))
-                        md["modified_datetime"] = datetime.fromtimestamp(float(md["modified_timestamp"])).strftime("%Y/%m/%d %H:%M:%S")
-                        files.append(md)
-                result.data = files
-                return result
+        try:
+            filesystems = await self.filesystems
+            command = f"find {filesystems.data[filesystem]['mountpoint']}/.zfs/snapshot -type f -name '{pattern}' -printf '%h\t%f\t%s\t%T@\n'"
+            result = await self.execute(command=command, notify=False, max_output_lines=1000)
+            files = []
+            for line in result.stdout_lines:
+                matches = re.match(
+                    "^(?P<location>[^\t]+)\t(?P<name>[^\t]+)\t(?P<bytes>[^\t]+)\t(?P<modified_timestamp>[^\n]+)",
+                    line,
+                )
+                if matches is not None:
+                    md = matches.groupdict()
+                    md["path"] = f"{md['location']}/{md['name']}"
+                    md["bytes"] = int(md["bytes"])
+                    md["size"] = format_bytes(md["bytes"])
+                    md["modified_datetime"] = datetime.fromtimestamp(float(md["modified_timestamp"])).strftime("%Y/%m/%d %H:%M:%S")
+                    md["modified_timestamp"] = float(md["modified_timestamp"])
+                    files.append(md)
+            result.data = files
+            return result
+        except KeyError:
+            pass
         return Result()
 
     @property
@@ -219,9 +222,11 @@ class Zfs:
                 matches = re.match("^(?P<filesystem>[^@]+)@(?P<name>[^\t]+)\t(?P<used_bytes>[^\t]+)\t(?P<creation>[^\t]+)\t(?P<userrefs>[^\n]+)", line)
                 if matches is not None:
                     md = matches.groupdict()
-                    md["creation_date"] = datetime.fromtimestamp(int(md["creation"])).strftime("%Y/%m/%d")
-                    md["creation_time"] = datetime.fromtimestamp(int(md["creation"])).strftime("%H:%M")
-                    md["used"] = format_bytes(int(md["used_bytes"]))
+                    md["used_bytes"] = int(md["used_bytes"])
+                    md["creation"] = int(md["creation"])
+                    md["creation_date"] = datetime.fromtimestamp(md["creation"]).strftime("%Y/%m/%d")
+                    md["creation_time"] = datetime.fromtimestamp(md["creation"]).strftime("%H:%M")
+                    md["used"] = format_bytes(md["used_bytes"])
                     snapshot = f"{md['filesystem']}@{md['name']}"
                     snapshots[snapshot] = md
             self._last_data[query] = snapshots
@@ -240,10 +245,10 @@ class Ssh(ssh.Ssh, Zfs):
     def notify(self, command: str):
         super().notify(f"<{self.host}> {command}")
 
-    async def execute(self, command: str, notify: bool = True) -> Result:
+    async def execute(self, command: str, max_output_lines: int = 0, notify: bool = True) -> Result:
         if notify:
             self.notify(command)
-        result = await super().execute(command)
+        result = await super().execute(command, max_output_lines)
         if result.stderr != "":
             el.notify(result.stderr, type="negative")
         result.name = self.host

bale/page.py

@@ -1,4 +1,5 @@
-from nicegui import app, ui
+import asyncio
+from nicegui import app, Client, ui
 from bale import elements as el
 from bale.drawer import Drawer
 from bale.content import Content
@@ -9,8 +10,8 @@ logger = logging.getLogger(__name__)
 
 
 def build():
-    @ui.page("/")
-    def page() -> None:
+    @ui.page("/", response_timeout=30)
+    async def index(client: Client) -> None:
         app.add_static_files("/static", "static")
         el.load_element_css()
         cli.load_terminal_css()
@@ -28,4 +29,4 @@ def build():
         content = Content()
         drawer = Drawer(column, content.host_selected, content.hide)
         drawer.build()
-        content.build()
+        await content.build()

bale/result.py

@@ -13,6 +13,7 @@ class Result:
     stdout_lines: List[str] = field(default_factory=list)
     stderr_lines: List[str] = field(default_factory=list)
     terminated: bool = False
+    truncated: bool = False
     data: Any = None
     trace: str = ""
     cached: bool = False

bale/tabs/automation.py

@@ -491,10 +491,10 @@ class Automation(Tab):
                 row.tailwind.width("[860px]").justify_content("center")
                 with ui.column() as col:
                     col.tailwind.height("full").width("[420px]")
-                    self.hosts = el.DSelect(source_hosts, label="Source Host(s)", multiple=True, with_input=True)
                     self.target_host = el.DSelect(target_host, label="Target Host", on_change=target_host_selected)
                     self.target_paths = [""]
-                    self.target_path = el.DSelect(self.target_paths, value="", label="Target Path", on_change=target_path_selected)
+                    self.target_path = el.DSelect(self.target_paths, value="", label="Target Path", new_value_mode="add-unique", on_change=target_path_selected)
+                    self.hosts = el.DSelect(source_hosts, label="Source Host(s)", multiple=True, with_input=True)
                     all_fs_to_lists()
                     with ui.scroll_area().classes("col"):
                         self.parentchildren = el.DSelect(
@@ -535,6 +535,8 @@ class Automation(Tab):
                 while target_path not in self.target_path.options and tries < 20:
                     await asyncio.sleep(0.1)
                     tries = tries + 1
+                if target_path not in self.target_paths:
+                    self.target_paths.append(target_path)
                 self.target_path.value = target_path
                 self.parentchildren.value = self.fs["values"].get("parentchildren", None)
                 self.parent.value = self.fs["values"].get("parent", None)

bale/tabs/history.py

@@ -1,7 +1,8 @@
 from datetime import datetime
 import json
-from . import SelectionConfirm, Tab
 from nicegui import ui, events
+import httpx
+from . import SelectionConfirm, Tab
 from bale import elements as el
 from bale.result import Result
 from bale.interfaces import zfs
@@ -47,8 +48,16 @@ class History(Tab):
                             "filter": "agTextColumnFilter",
                             "flex": 1,
                         },
-                        {"headerName": "Date", "field": "date", "filter": "agDateColumnFilter", "maxWidth": 100},
-                        {"headerName": "Time", "field": "time", "maxWidth": 100},
+                        {
+                            "headerName": "Timestamp",
+                            "field": "timestamp",
+                            "filter": "agTextColumnFilter",
+                            "maxWidth": 200,
+                            ":cellRenderer": """(data) => {
+                                var date = new Date(data.value * 1000).toLocaleString(undefined, {dateStyle: 'short', timeStyle: 'short', hour12: false});;
+                                return date;
+                            }""",
+                        },
                         {
                             "headerName": "Status",
                             "field": "status",
@@ -90,6 +99,20 @@ class History(Tab):
             http[status]["data"] = e.content["json"]["data"]
             http[status]["headers"] = e.content["json"]["headers"]
 
+        def test(status):
+            try:
+                url = http[status]["url"]
+                data = self.process_pipe_data(result=Result(name=self.host, command="TEST COMMAND", status=status), data=http[status]["data"])
+                headers = http[status]["headers"]
+                post = httpx.post(url=url, json=data, headers=headers)
+                print(post.status_code)
+                if post.status_code == 200:
+                    el.notify("Test successful!", type="positive")
+                else:
+                    el.notify(f"Test failed with status code {post.status_code}!", type="negative")
+            except:
+                el.notify("Test failed!", type="negative")
+
         def show_controls(status):
             if status not in http:
                 http[status] = {}
@@ -107,7 +130,7 @@ class History(Tab):
                     "topic": "mytopic",
                     "tags": ["turtle"],
                     "title": "Successful Automation Run for {name}",
-                    "message": "{stdout}",
+                    "message": "{command}",
                 },
             )
             editor.properties["content"]["json"]["headers"] = self.get_pipe_status("http", status).get("headers", {"Authorization": "Bearer tk_..."})
@@ -126,11 +149,15 @@ class History(Tab):
                     with ui.step("On Success"):
                         with el.WColumn().classes("col justify-start"):
                             show_controls(status="success")
-                        el.LgButton("NEXT", on_click=lambda _: stepper.next())
+                        with el.WRow():
+                            el.LgButton("TEST", on_click=lambda _: test(status="success"))
+                            el.LgButton("NEXT", on_click=lambda _: stepper.next())
                     with ui.step("On Error"):
                         with el.WColumn().classes("col justify-start"):
                             show_controls(status="error")
-                        el.DButton("SAVE", on_click=lambda: host_dialog.submit("save"))
+                        with el.WRow():
+                            el.LgButton("TEST", on_click=lambda _: test(status="error"))
+                            el.DButton("SAVE", on_click=lambda: host_dialog.submit("save"))
 
         result = await host_dialog
         if result == "save":

bale/tabs/manage.py

@@ -1,6 +1,6 @@
 import asyncio
 from copy import deepcopy
-from nicegui import ui
+from nicegui import background_tasks, ui
 from . import SelectionConfirm, Tab, Task
 from bale.result import Result
 from bale import elements as el
@@ -75,11 +75,19 @@ class Manage(Tab):
                             "field": "used",
                             "maxWidth": 100,
                             ":comparator": """(valueA, valueB, nodeA, nodeB, isInverted) => {
-                                                return (nodeA.data.used_bytes > nodeB.data.used_bytes) ? -1 : 1;
-                                            }""",
+                                return (nodeA.data.used_bytes > nodeB.data.used_bytes) ? -1 : 1;
+                            }""",
+                        },
+                        {
+                            "headerName": "Created",
+                            "field": "creation",
+                            "filter": "agTextColumnFilter",
+                            "maxWidth": 200,
+                            ":cellRenderer": """(data) => {
+                                var date = new Date(data.value * 1000).toLocaleString(undefined, {dateStyle: 'short', timeStyle: 'short', hour12: false});;
+                                return date;
+                            }""",
                         },
-                        {"headerName": "Creation Date", "field": "creation_date", "filter": "agDateColumnFilter", "maxWidth": 150},
-                        {"headerName": "Creation Time", "field": "creation_time", "maxWidth": 150},
                         {"headerName": "Holds", "field": "userrefs", "filter": "agNumberColumnFilter", "maxWidth": 100},
                     ],
                     "rowData": [],
@@ -92,8 +100,8 @@ class Manage(Tab):
         self._spinner.visible = True
         self.zfs.invalidate_query()
         snapshots = await self.zfs.snapshots
-        await self.zfs.filesystems
-        await self.zfs.holds_for_snapshot()
+        background_tasks.create(self.zfs.filesystems, name="zfs_filesystems")
+        background_tasks.create(self.zfs.holds_for_snapshot(), name="zfs_holds")
         self._grid.options["rowData"] = list(snapshots.data.values())
         self._grid.update()
         self._spinner.visible = False
@@ -103,9 +111,12 @@ class Manage(Tab):
         result = await SelectionConfirm(container=self._confirm, label=">BROWSE<")
         if result == "confirm":
             rows = await self._grid.get_selected_rows()
-            filesystems = await self.zfs.filesystems
-            mount_path = filesystems.data[rows[0]["filesystem"]]["mountpoint"]
-            await sshdl.SshFileBrowse(zfs=self.zfs, path=f"{mount_path}/.zfs/snapshot/{rows[0]['name']}")
+            try:
+                filesystems = await self.zfs.filesystems
+                mount_path = filesystems.data[rows[0]["filesystem"]]["mountpoint"]
+                await sshdl.SshFileBrowse(zfs=self.zfs, path=f"{mount_path}/.zfs/snapshot/{rows[0]['name']}")
+            except KeyError:
+                el.notify(f"Unable to browse {rows[0]['filesystem']}", type="warning")
         self._set_selection()
 
     async def _find(self) -> None: