Merge pull request #19 from downtownallday/browser-tests

Add Roundcube QA tests

tests/lib/python/browser/NcContactsAutomation.py

@@ -0,0 +1,60 @@
+#####
+##### This file is part of Mail-in-a-Box-LDAP which is released under the
+##### terms of the GNU Affero General Public License as published by the
+##### Free Software Foundation, either version 3 of the License, or (at
+##### your option) any later version. See file LICENSE or go to
+##### https://github.com/downtownallday/mailinabox-ldap for full license
+##### details.
+#####
+
+from selenium.common.exceptions import (
+    NoSuchElementException,
+)
+
+class NcContactsAutomation(object):
+    def __init__(self, nc):
+        ''' `nc` is a NextcloudAutomation object '''
+        self.nc = nc
+        self.d = nc.d
+
+    def click_contact(self, contact):
+        d = self.d
+        d.say("Click contact %s", contact['email'])
+        found = False
+        # .list-item-content (nc 25+)
+        # .option__details (nc <25)
+        els = d.find_els('div.contacts-list div.list-item-content,div.option__details')
+        d.say_verbose('found %s contacts' % len(els))
+        for el in els:
+            # .line-one (nc 25+)
+            # .option__lineone (nc <25)
+            fullname = el.find_el('.line-one,.option__lineone').content().strip()
+            email = el.find_el('.line-two,.option__linetwo').content().strip()
+            d.say_verbose('contact: "%s" <%s>', fullname, email)
+            if fullname.lower() == "%s %s" % (contact['givenname'].lower(), contact['surname'].lower()) and email.lower() == contact['email'].lower():
+                found = True
+                el.click()
+                break
+        if not found: raise NoSuchElementException()
+
+    def wait_contact_loaded(self, secs=5):
+        d = self.d
+        d.say("Wait for contact to load")
+        d.wait_for_el('section.contact-details', secs=secs)
+        
+    def delete_current_contact(self):
+        d = self.d
+        d.say("Delete current contact")
+        # Click ... menu
+        d.find_el('.contact-header__actions button.action-item__menutoggle').click()
+        # .v-popper__popper (nc 25+)
+        # .popover (nc <25)
+        el = d.wait_for_el(
+            '.v-popper__popper,.popover',
+            must_be_displayed=True,
+            secs=2
+        )
+        # click "delete"
+        # .delete-icon (nc 25+)
+        # .icon-delete (nc <25)
+        delete = el.find_el('span.delete-icon,span.icon-delete').click()

tests/lib/python/browser/NextcloudAutomation.py

@@ -0,0 +1,78 @@
+#####
+##### This file is part of Mail-in-a-Box-LDAP which is released under the
+##### terms of the GNU Affero General Public License as published by the
+##### Free Software Foundation, either version 3 of the License, or (at
+##### your option) any later version. See file LICENSE or go to
+##### https://github.com/downtownallday/mailinabox-ldap for full license
+##### details.
+#####
+
+from selenium.common.exceptions import (
+    NoSuchElementException,
+)
+from .NcContactsAutomation import NcContactsAutomation
+
+class NextcloudAutomation(object):
+    def __init__(self, d):
+        ''' `d` is a browser.automation TestDriver object '''
+        self.d = d
+        
+    def wait_for_login_screen(self, secs=7):
+        d = self.d
+        d.say("Wait for login screen")
+        d.wait_for_el('form[name=login] input#user', secs=secs)
+
+    def login(self, login, pw):
+        d = self.d
+        d.say("Login %s to Nextcloud", login)
+        d.find_el('input#user').send_text(login)
+        d.find_el('input#password').send_text(pw)
+        submit = d.find_el('button[type="submit"]', throws=False)
+        # submit button for nextcloud < 25 (jquery)
+        if not submit: submit = d.find_el('#submit-wrapper') # nc<25
+        submit.click()
+
+    def logout(self):
+        d = self.d
+        d.say("Logout of Nextcloud")
+        d.find_el('#settings .avatardiv').click()
+        d.find_el('[data-id="logout"] a').click()
+
+    def open_contacts(self):
+        d = self.d
+        d.say("Open contacts")
+        # nc 25+
+        el = d.find_el('header [data-app-id="contacts"]', throws=False)
+        if not el:
+            # nc < 25
+            el = d.find_el('header [data-id="contacts"]')
+        self.close_first_run_wizard()
+        el.click()
+        return NcContactsAutomation(self)
+
+    def wait_for_app_load(self, secs=7):
+        d = self.d
+        d.say("Wait for app to load")
+
+        # some apps are vue, some jquery (legacy)
+        vue = d.find_el('#app-content-vue', throws=False)
+        if not vue: vue = d.find_el('#app-dashboard', throws=False)
+        jquery = d.find_el('#app-content', throws=False)
+        
+        if vue:
+            d.wait_tick(1000)
+            
+        elif jquery:
+            d.say_verbose('Waiting on a jquery app')
+            d.wait_until_true('return window.$.active == 0', secs=secs)
+            
+        else:            
+            raise NoSuchElementException('#app-dashboard, #app-content or #app-content-vue')
+
+    def close_first_run_wizard(self):
+        d = self.d
+        firstrunwiz = d.find_el('#firstrunwizard', throws=False, quiet=True)
+        if firstrunwiz and firstrunwiz.is_displayed():
+            d.say_verbose("closing first run wizard")
+            d.find_el('#firstrunwizard span.close-icon').click()
+            d.wait_tick(1)

tests/lib/python/browser/RoundcubeAutomation.py

@@ -0,0 +1,63 @@
+#####
+##### This file is part of Mail-in-a-Box-LDAP which is released under the
+##### terms of the GNU Affero General Public License as published by the
+##### Free Software Foundation, either version 3 of the License, or (at
+##### your option) any later version. See file LICENSE or go to
+##### https://github.com/downtownallday/mailinabox-ldap for full license
+##### details.
+#####
+
+
+class RoundcubeAutomation(object):
+    def __init__(self, d):
+        ''' `d` is a browser.automation TestDriver object '''
+        self.d = d
+        
+    def wait_for_login_screen(self, secs=5):
+        d = self.d
+        d.say("Wait for login screen")
+        d.wait_for_el('#rcmloginuser', secs=secs)
+
+    def login(self, login, pw):
+        d = self.d
+        d.say("Login %s to roundcube", login)
+        d.find_el('#rcmloginuser').send_text(login)
+        d.find_el('#rcmloginpwd').send_text(pw)
+        d.find_el('#rcmloginsubmit').click()
+        
+    def logout(self):
+        d = self.d
+        d.say("Logout of roundcube")
+        el = d.wait_for_el('a.logout', must_be_enabled=True).click()
+
+    def open_inbox(self):
+        d = self.d
+        d.say("Open inbox")
+        d.find_el('a.mail').click()
+        
+    def wait_for_inbox(self, secs=10):
+        d = self.d
+        d.say("Wait for inbox")
+        d.wait_for_el('body.task-mail')
+        d.wait_for_el('a.logout', must_be_enabled=True, secs=secs)
+
+    def open_settings(self):
+        d = self.d
+        d.say("Open settings")
+        d.find_el('a.settings').click()
+
+    def wait_for_settings(self, secs=10):
+        d = self.d
+        d.say("Wait for settings")
+        d.wait_for_el('body.task-settings', secs=secs)
+
+    def open_contacts(self):
+        d = self.d
+        d.say("Open contacts")
+        d.find_el('a.contacts').click()
+
+    def wait_for_contacts(self, secs=10):
+        d = self.d
+        d.say("Wait for contacts")
+        d.wait_for_el('body.task-addressbook', secs=secs)
+

tests/lib/python/browser/automation.py

@@ -0,0 +1,549 @@
+#####
+##### This file is part of Mail-in-a-Box-LDAP which is released under the
+##### terms of the GNU Affero General Public License as published by the
+##### Free Software Foundation, either version 3 of the License, or (at
+##### your option) any later version. See file LICENSE or go to
+##### https://github.com/downtownallday/mailinabox-ldap for full license
+##### details.
+#####
+
+from selenium.webdriver import (
+    Chrome,
+    ChromeOptions,
+    Firefox,
+    FirefoxOptions
+)
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.remote.webelement import WebElement
+from selenium.webdriver.common.by import By
+from selenium.common.exceptions import (
+    NoSuchElementException,
+    TimeoutException,
+    ElementNotInteractableException
+)
+
+import os
+import subprocess
+import time
+
+
+#
+# chrome:
+#    snap install chromium
+#
+# firefox:
+#    apt-get install firefox
+#    get the latest compiled geckodriver from:
+#        https://github.com/mozilla/geckodriver/releases
+#    copy into /usr/local/bin
+#
+# all:
+#    pip3 install selenium (python 3.7 is required by selenium)
+#
+
+# OLD: for headless firefox (before firefox supported headless)
+#    apt-get -y install xorg xvfb gtk2-engines-pixbuf
+#    apt-get -y install dbus-x11 xfonts-base xfonts-100dpi xfonts-75dpi xfonts-cyrillic xfonts-scalable
+#    apt-get -y install imagemagick x11-apps
+#
+#    before running tests, create an X frame buffer display:
+#       Xvfb -ac :99 -screen 0 1280x1024x16 & export DISPLAY=:99
+#
+
+
+class ChromeTestDriver(Chrome):
+    def __init__(self, options=None):
+        '''Initialze headless chrome. If problems arise, try running from the
+           command line: `chromium --headless http://localhost/mail/`
+
+        '''
+        if not options:
+            options = ChromeOptions()
+            options.headless = True
+            
+            # set a window size
+            options.add_argument("--window-size=1200x600")
+            
+            # deal with ssl certificates since chrome has its own
+            # trusted ca list and does not use the system's
+            options.add_argument('--allow-insecure-localhost')
+            options.add_argument('--ignore-certificate-errors')
+
+            # required to run chromium as root
+            options.add_argument('--no-sandbox')
+            
+        super(ChromeTestDriver, self).__init__(
+            executable_path='/snap/bin/chromium.chromedriver',
+            options=options
+        )
+
+        self.delete_all_cookies()
+
+
+class FirefoxTestDriver(Firefox):
+    ''' TODO: untested '''
+    def __init__(self, options=None):
+        if not options:
+            options = FirefoxOptions()
+            options.headless = True
+            
+        super(FirefoxTestDriver, self).__init__(
+            executable_path='/usr/local/bin/geckodriver',
+            options=options
+        )
+
+        self.delete_all_cookies()
+
+
+class TestDriver(object):
+    def __init__(self, driver=None, verbose=None, base_url=None, output_path=None):
+        self.first_start_time = None
+        self.start_time = None
+        self.start_msg = []
+        self.next_tick_id = 0
+        
+        if driver is None:
+            if 'BROWSER_TESTS_BROWSER' in os.environ:
+                driver = os.environ['BROWSER_TESTS_BROWSER']
+            else:
+                driver = 'chrome'
+        if isinstance(driver, str):
+            driver = TestDriver.createByName(driver)
+        self.driver = driver
+        
+        if verbose is None:
+            if 'BROWSER_TESTS_VERBOSITY' in os.environ:
+                verbose = int(os.environ['BROWSER_TESTS_VERBOSITY'])
+            else:
+                verbose = 1
+        self.verbose = verbose
+        
+        if base_url is None:
+            if 'BROWSER_TESTS_BASE_URL' in os.environ:
+                base_url = os.environ['BROWSER_TESTS_BASE_URL']
+            else:
+                hostname = subprocess.check_output(['/bin/hostname','--fqdn'])
+                base_url = "https://%s" % hostname.decode('utf-8').strip()
+        self.base_url = base_url
+
+        if output_path is None:
+            if 'BROWSER_TESTS_OUTPUT_PATH' in os.environ:
+                output_path = os.environ['BROWSER_TESTS_OUTPUT_PATH']
+            else:
+                output_path= "./"
+        self.output_path = output_path
+
+        
+
+    @staticmethod
+    def createByName(name):
+        if name == 'chrome':
+            return ChromeTestDriver()
+        elif name == 'firefox':
+            return FirefoxTestDriver()
+        raise ValueError('no such driver named "%s"' % name)
+
+    def _say(self, loglevel, heirarchy_level, *args):
+        if self.verbose >= loglevel:
+            for i in range(len(self.start_msg), heirarchy_level+1):
+                self.start_msg.append(None)
+            self.start_msg = self.start_msg[0:heirarchy_level]
+            indent = 0
+            for item in self.start_msg:
+                if item is not None: indent += 1
+            msg = args[0] % (args[1:])
+            self.start_msg.append(msg)
+            print('  '*indent + msg + ' ')
+            
+            
+    def is_verbose(self):
+        return self.verbose >= 2
+        
+    def say_verbose(self, *args):
+        self._say(2, 2, *args)
+
+    def say(self, *args):
+        self._say(1, 1, *args)
+
+    def start(self, *args):
+        now = time.time()
+        if self.start_time is not None:
+            elapsed = format(now - self.start_time, '.1f')
+            self._say(2, 0, '[%s: %s seconds]\n', self.start_msg[0], elapsed)
+        else:
+            self.first_start_time = now
+        self.start_time = now
+        self._say(1, 0, *args)
+    
+    def last_start(self):
+        msg = []
+        for item in self.start_msg:
+            if item is not None: msg.append(item)
+        return " / ".join(msg)
+
+    
+    def get(self, url):
+        ''' load a web page in the current browser session '''
+        if not url.startswith('http'):
+            url = self.base_url + url
+        self.say_verbose('get %s', url)
+        self.driver.get(url)
+        return self
+
+    def title(self):
+        return self.driver.title
+
+    def current_url(self):
+        return self.driver.current_url
+
+    def refresh(self):
+        self.driver.refresh()
+        return self
+
+    def get_current_window_handle(self):
+        ''' returns the string id of the current window/tab '''
+        return self.driver.current_window_handle
+    
+    def get_window_handles(self):
+        ''' returns an array of strings, one for each window or tab open '''
+        return self.driver.window_handles
+
+    def switch_to_window(self, handle):
+        ''' returns the current window handle '''
+        cur = self.get_current_window_handle()
+        self.driver.switch_to.window(handle)
+        return cur
+
+    def switch_to_frame(self, iframe_el):
+        self.driver.switch_to.frame(iframe_el.el)
+
+    def switch_to_parent_frame(self, iframe_el):
+        # untested
+        self.driver.switch_to.parent_frame(iframe_el.el)
+
+    def save_screenshot(self, where, ignore_errors=False, quiet=True):
+        ''' where - path and file name of screen shotfile 
+            eg: "out/screenshot.png". '''
+        if not where.startswith('/'):
+            where = os.path.join(self.output_path, where)
+        try:
+            os.makedirs(os.path.dirname(where), exist_ok=True)
+            self._say(1 if quiet else 2, 2, "save screenshot: '%s'", where)
+            self.driver.save_screenshot(where)
+        except Exception as e:
+            if not ignore_errors:
+                raise e
+
+    def delete_cookie(self, name):
+        self.driver.delete_cookie(name)
+            
+    def wait_for_id(self, id, secs=5, throws=True):
+        return self.wait_for_el('#' + id, secs=secs, throws=throws)
+
+    def wait_for_el(self, css_selector, secs=5, throws=True, must_be_enabled=False, must_be_displayed=None):
+        msg=[]
+        if must_be_enabled:
+            msg.append('enabled')
+        if must_be_displayed is True:
+            msg.append('displayed')
+        elif must_be_displayed is False:
+            msg.append('hidden')
+        if len(msg)==0:
+            self.say_verbose("wait for selector '%s' (%ss)",
+                             css_selector, secs)
+        else:
+            self.say_verbose("wait for selector '%s' to be %s (%ss)",
+                             css_selector, ",".join(msg), secs)
+        def test_fn(driver):
+            found_el = driver.find_element(By.CSS_SELECTOR, css_selector)
+            if must_be_enabled and not found_el.is_enabled():
+                raise NoSuchElementException()
+            if must_be_displayed is not None:
+                if must_be_displayed and not found_el.is_displayed():
+                    raise NoSuchElementException()
+                if not must_be_displayed and found_el.is_displayed():
+                    raise NoSuchElementException()                
+            return found_el
+        wait = WebDriverWait(self.driver, secs, ignored_exceptions= (
+            NoSuchElementException
+        ))
+        try:
+            rtn = wait.until(test_fn)
+            return ElWrapper(self, rtn)
+        except TimeoutException as e:
+            if throws: raise e
+            else: return None
+
+    def wait_for_el_not_exists(self, css_selector, secs=5, throws=True):
+        self.say_verbose("wait for selector '%s' (%ss) to not exist",
+                             css_selector, secs)
+        def test_fn(driver):
+            found_el = driver.find_element(By.CSS_SELECTOR, css_selector)
+            if found_el: raise NoSuchElementException()
+        wait = WebDriverWait(self.driver, secs, ignored_exceptions= (
+            NoSuchElementException
+        ))
+        try:
+            wait.until(test_fn)
+            return True
+        except TimeoutException as e:
+            if throws: raise e
+            else: return None
+
+    def wait_for_text(self, text, tag='*', secs=5, exact=False, throws=True, case_sensitive=False):
+        self.say_verbose("wait for text '%s'", text)
+        def test_fn(driver):
+            return self.find_text(text, tag=tag, exact=exact, throws=False, quiet=True, case_sensitive=case_sensitive)
+        wait = WebDriverWait(self.driver, secs, ignored_exceptions= (
+            NoSuchElementException
+        ))
+        try:
+            rtn = wait.until(test_fn)
+            return rtn
+        except TimeoutException as e:
+            if throws: raise e
+            else: return None
+
+    def find_el(self, css_selector, nth=0, throws=True, quiet=False):
+        try:
+            els = self.driver.find_elements(By.CSS_SELECTOR, css_selector)
+            if len(els)==0:
+                if not quiet: self.say_verbose("find element: '%s' (not found)", css_selector)
+                raise NoSuchElementException("selector=%s" % css_selector)
+            if not quiet: self.say_verbose("find element: '%s' (returning #%s/%s)", css_selector, nth+1, len(els))
+            return ElWrapper(self, els[nth])
+        except (IndexError, NoSuchElementException) as e:
+            if throws: raise e
+            else: return None
+
+    def find_els(self, css_selector, throws=True, displayed=False):
+        self.say_verbose("find elements: '%s'", css_selector)
+        try:
+            els = self.driver.find_elements(By.CSS_SELECTOR, css_selector)
+            return [ ElWrapper(self, el) for el in els if not displayed or el.is_displayed() ]
+        except (IndexError, NoSuchElementException) as e:
+            if throws: raise e
+            else: return None
+
+    def find_text(self, text, tag='*', exact=False, throws=True, quiet=False, case_sensitive=False):
+        if not quiet:
+            self.say_verbose("find text: '%s' tag=%s exact=%s",
+                             text, tag, exact)
+        try:
+            if exact:
+                if case_sensitive:
+                    xpath = "//%s[normalize-space(text()) = '%s']" % (tag, text)
+                else:
+                    uc = text.upper()
+                    lc = text.lower()
+                    xpath = "//%s[normalize-space(translate(text(), '%s', '%s')) = '%s']" % (tag, lc, uc, uc)
+            else:
+                if case_sensitive:
+                    xpath = "//%s[contains(text(),'%s')]" % (tag, text)
+                else:
+                    uc = text.upper()
+                    lc = text.lower()
+                    xpath = "//%s[contains(translate(text(),'%s','%s'),'%s')]" % (tag, lc, uc, uc)
+
+            el = self.driver.find_element(by=By.XPATH, value=xpath)
+            return ElWrapper(self, el)
+        except NoSuchElementException as e:
+            if throws: raise e
+            else: return None
+
+    def sleep(self, secs):
+        self.say_verbose('sleep %s secs', secs)
+        def test_fn(driver):
+            raise NoSuchElementException
+        wait = WebDriverWait(self.driver, secs, ignored_exceptions= (
+            NoSuchElementException
+        ))
+        try:
+            wait.until(test_fn)
+        except TimeoutException as e:
+            pass
+
+    def execute_script(self, script, quiet=False, *args):
+        ''' Synchronously Executes JavaScript in the current window/frame '''
+        newargs = []
+        for arg in args:
+            if isinstance(arg, ElWrapper): newargs.append(arg.el)
+            else: newargs.append(arg)
+        if not quiet:
+            self.say_verbose('execute script: %s', script.replace('\n',' '))
+        return self.driver.execute_script(script, *newargs)
+
+    def execute_async_script(self, script, secs=5, *args):
+        ''' Asynchronously Executes JavaScript in the current window/frame '''
+        self.driver.set_script_timeout(secs)
+        self.driver.execute_async_script(script, *args)
+
+    def wait_until_true(self, script, secs=5, *args):
+        self.say_verbose('run script until true: %s', script)
+        d = self
+        class NotTrue(Exception):
+            pass
+        def test_fn(driver):
+            nonlocal script, args
+            p = driver.execute_script(script, quiet=True, *args)
+            driver.say_verbose("script returned: %s", p)
+            if not p: raise NotTrue()
+            return True
+        wait = WebDriverWait(self, secs, ignored_exceptions= (
+            NotTrue
+        ))
+        wait.until(test_fn)  # throws TimeoutException
+
+    def wait_tick(self, delay_ms, secs=5):
+        # allow time for vue to render (delay_ms>=1)
+        cancel_id = self.execute_script('window.qa_ticked=false; return window.setTimeout(() => { window.qa_ticked=true; }, %s)' % delay_ms);
+        self.wait_until_true('return window.qa_ticked === true', secs=secs)
+    
+    def close(self):
+        ''' close the window/tab '''
+        self.say_verbose("closing %s", self.driver.current_url)
+        self.driver.close()
+
+    def quit(self):
+        ''' closes the browser and shuts down the chromedriver executable '''
+        now = time.time()
+        if self.first_start_time is not None:
+            elapsed = format(now - self.first_start_time, '.1f')
+            self._say(2, 0, '[TOTAL TIME: %s seconds]\n', elapsed)
+        self.driver.quit()
+
+    def fail(self, exception):
+        last_start = self.last_start()
+        self.start("Failure!")
+        self.save_screenshot('screenshot.png', ignore_errors=False, quiet=False)
+        if hasattr(exception, 'msg') and exception.msg != '':
+            exception.msg = "Error during '%s': %s" % (last_start, exception.msg)
+        else:
+            exception.msg = "Error during '%s'" % last_start
+
+    
+
+
+class ElWrapper(object):
+    '''see:
+        https://github.com/SeleniumHQ/selenium/blob/trunk/py/selenium/webdriver/remote/webelement.py
+
+    '''
+    def __init__(self, driver, el):
+        self.driver = driver
+        self.el = el
+
+    def find_el(self, css_selector, nth=0, throws=True, quiet=False):
+        try:
+            els = self.el.find_elements(By.CSS_SELECTOR, css_selector)
+            if len(els)==0:
+                if not quiet: self.driver.say_verbose("find element: '%s' (not found)", css_selector)
+                raise NoSuchElementException("selector=%s" % css_selector)
+            if not quiet: self.driver.say_verbose("find element: '%s' (returning #%s/%s)", css_selector, nth+1, len(els))
+            return ElWrapper(self.driver, els[nth])
+        except (IndexError, NoSuchElementException) as e:
+            if throws: raise e
+            else: return None
+
+    def find_els(self, css_selector, throws=True, displayed=False):
+        self.driver.say_verbose("find elements: '%s'", css_selector)
+        try:
+            els = self.el.find_elements(By.CSS_SELECTOR, css_selector)
+            return [ ElWrapper(self.driver, el) for el in els if not displayed or el.is_displayed() ]
+        except (IndexError, NoSuchElementException) as e:
+            if throws: raise e
+            else: return None
+    
+    def is_enabled(self):
+        return self.el.is_enabled()
+
+    def is_checked(self):
+        """ a checkbox or radio button is checked """
+        return self.el.is_selected()
+
+    def is_displayed(self):
+        """Whether the self.element is visible to a user."""
+        return self.el.is_displayed()
+    
+    def get_attribute(self, name):
+        return self.el.get_attribute(name)
+
+    def get_property(self, expr):
+        self.driver.say_verbose('get property %s', expr)
+        prefix = '.'
+        if expr.startswith('.') or expr.startswith('['): prefix=''
+        p = self.driver.execute_script(
+            "return arguments[0]%s%s;" % ( prefix, expr ),
+            self.el
+        )
+        if isinstance(p, WebElement):
+            p = ElWrapper(self.driver, p)
+        if isinstance(p, bool):
+            self.driver.say_verbose('property result: %s', p)
+        else:
+            self.driver.say_verbose('property result: %s', p.__class__)
+        return p
+
+    def content(self, max_length=None, ellipses=True):
+        txt = self.el.text
+        if not max_length or len(txt)<max_length:
+            return txt
+        if ellipses:
+            return txt[0:max_length] + '...'
+        return txt[0:max_length]
+
+    def tag(self):
+        return self.el.tag_name
+
+    def location(self):
+        """ returns dictionary {x:N, y:N} """
+        return self.el.location()
+
+    def rect(self):
+        return self.el.rect()
+
+    def parent(self):
+        # get the parent element
+        p = self.driver.execute_script(
+            "return arguments[0].parentNode;",
+            self.el
+        )
+        return ElWrapper(self.driver, p)
+    
+    def send_text(self, *value):
+        self.driver.say_verbose("send text '%s'", "/".join(value))
+        self.send_keys(*value)
+        return self
+        
+    def send_keys(self, *value):
+        self.el.send_keys(*value)
+        return self
+
+    def clear_text(self):
+        self.el.clear()
+        return self
+
+    def click(self):
+        if self.driver.is_verbose():
+            content = self.content(max_length=40).replace('\n',' ').strip()
+            tag = self.tag()
+            if tag=='a':
+                tag='link'
+                if content == '': content=self.get_attribute('href')
+            if content != '':
+                self.driver.say_verbose("click %s '%s'", tag, content)
+            else:
+                self.driver.say_verbose("click %s", tag)            
+        self.el.click()
+        return self
+
+        
+
+
+#dir(el)
+#['__abstractmethods__', '__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', '__weakref__', '_abc_impl', '_execute', '_id', '_parent', '_upload', 'accessible_name', 'aria_role', 'clear', 'click', 'find_element', 'find_elements', 'get_attribute', 'get_dom_attribute', 'get_property', 'id', 'is_displayed', 'is_enabled', 'is_selected', 'location', 'location_once_scrolled_into_view', 'parent', 'rect', 'screenshot', 'screenshot_as_base64', 'screenshot_as_png', 'send_keys', 'shadow_root', 'size', 'submit', 'tag_name', 'text', 'value_of_css_property']
+
+        
+#dir(driver)
+#['__abstractmethods__', '__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__enter__', '__eq__', '__exit__', '__format__', '__ge__', '__getattribute__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', '__weakref__', '_abc_impl', '_authenticator_id', '_file_detector', '_get_cdp_details', '_is_remote', '_mobile', '_shadowroot_cls', '_switch_to', '_unwrap_value', '_web_element_cls', '_wrap_value', 'add_cookie', 'add_credential', 'add_virtual_authenticator', 'application_cache', 'back', 'bidi_connection', 'capabilities', 'caps', 'close', 'command_executor', 'create_options', 'create_web_element', 'current_url', 'current_window_handle', 'delete_all_cookies', 'delete_cookie', 'delete_network_conditions', 'desired_capabilities', 'error_handler', 'execute', 'execute_async_script', 'execute_cdp_cmd', 'execute_script', 'file_detector', 'file_detector_context', 'find_element', 'find_elements', 'forward', 'fullscreen_window', 'get', 'get_cookie', 'get_cookies', 'get_credentials', 'get_issue_message', 'get_log', 'get_network_conditions', 'get_pinned_scripts', 'get_screenshot_as_base64', 'get_screenshot_as_file', 'get_screenshot_as_png', 'get_sinks', 'get_window_position', 'get_window_rect', 'get_window_size', 'implicitly_wait', 'launch_app', 'log_types', 'maximize_window', 'minimize_window', 'mobile', 'name', 'orientation', 'page_source', 'pin_script', 'pinned_scripts', 'port', 'print_page', 'quit', 'refresh', 'remove_all_credentials', 'remove_credential', 'remove_virtual_authenticator', 'save_screenshot', 'service', 'session_id', 'set_network_conditions', 'set_page_load_timeout', 'set_permissions', 'set_script_timeout', 'set_sink_to_use', 'set_user_verified', 'set_window_position', 'set_window_rect', 'set_window_size', 'start_client', 'start_desktop_mirroring', 'start_session', 'start_tab_mirroring', 'stop_casting', 'stop_client', 'switch_to', 'timeouts', 'title', 'unpin', 'vendor_prefix', 'virtual_authenticator_id', 'window_handles']
+

tests/runner.sh

@@ -31,6 +31,7 @@ default_suites=(
 	mail-access
 	management-users
 	z-push
+	roundcube     # browser tests
 )
 
 extra_suites=(

tests/suites/.gitignore

@@ -0,0 +1 @@
+screenshot.png

tests/suites/_init_miabldap.sh

@@ -17,11 +17,14 @@ set +eu
 . suites/_mail-functions.sh || exit 1
 . suites/_mgmt-functions.sh || exit 1
 . suites/_zpush-functions.sh || exit 1
+. suites/_ui-functions.sh || exit 1
 
 
 MIAB_DIR=".."
 PYMAIL="./test_mail.py"
 EDITCONF="../tools/editconf.py"
+UI_TESTS_PYTHONPATH=$(realpath "lib/python")
+UI_TESTS_VERBOSITY=2
 
 # options
 SKIP_REMOTE_SMTP_TESTS=no

tests/suites/_ui-functions.sh

@@ -0,0 +1,44 @@
+#####
+##### This file is part of Mail-in-a-Box-LDAP which is released under the
+##### terms of the GNU Affero General Public License as published by the
+##### Free Software Foundation, either version 3 of the License, or (at
+##### your option) any later version. See file LICENSE or go to
+##### https://github.com/downtownallday/mailinabox-ldap for full license
+##### details.
+#####
+
+run_browser_test() {
+    local assert=false
+    if [ "$1" = "assert" ]; then
+        assert=true
+        shift
+    fi
+    local path="$1" # relative to suites directory. eg "roundcube/mytest.py"
+    shift;  # remaining arguments are passed to the test
+    
+    record "[launching ui test $path $*]"
+    record "PYTHONPATH=$UI_TESTS_PYTHONPATH"
+    record "BROWSER_TESTS_VERBOSITY=${UI_TESTS_VERBOSITY:-1}"
+    record "BROWSER_TESTS_OUTPUT_PATH=${TEST_OF}_ui"
+    
+    local output
+    output=$(
+        export PYTHONPATH="$UI_TESTS_PYTHONPATH";
+        export BROWSER_TESTS_VERBOSITY=${UI_TESTS_VERBOSITY:-1};
+        export BROWSER_TESTS_OUTPUT_PATH="${TEST_OF}_ui";
+        python3 suites/$path "$@" 2>&1
+          )
+    local code=$?
+    record "RESULT: $code"
+    record "OUTPUT:"; record "$output"
+    if [ $code -ne 0 ] && $assert; then
+        test_failure "ui test failed: $(python_error "$output")"
+    fi
+    return $code
+}
+
+
+assert_browser_test() {
+    run_browser_test "assert" "$@"
+    return $?
+}

tests/suites/nextcloud/contacts.py

@@ -0,0 +1,89 @@
+#####
+##### This file is part of Mail-in-a-Box-LDAP which is released under the
+##### terms of the GNU Affero General Public License as published by the
+##### Free Software Foundation, either version 3 of the License, or (at
+##### your option) any later version. See file LICENSE or go to
+##### https://github.com/downtownallday/mailinabox-ldap for full license
+##### details.
+#####
+
+from browser.automation import (
+    TestDriver,
+    TimeoutException,
+    NoSuchElementException,
+    ElementNotInteractableException,
+)
+from browser.NextcloudAutomation import NextcloudAutomation
+import sys
+
+op = sys.argv[1]
+login = sys.argv[2]
+pw = sys.argv[3]
+contact = {
+    'givenname': sys.argv[4],
+    'surname': sys.argv[5],
+    'email': sys.argv[6],
+}
+
+d = TestDriver()
+nc = NextcloudAutomation(d)
+
+try:
+    #
+    # open the browser to Nextcloud
+    #
+    # these tests work for both remote and local Nextclouds. nginx
+    # will redirect to a remote nextcloud during get(), if configured
+    #
+    d.start("Opening Nextcloud")
+    d.get("/cloud/")
+    nc.wait_for_login_screen()
+    d.say_verbose('url: ' + d.current_url())
+
+    #
+    # login
+    #
+    nc.login(login, pw)
+    nc.wait_for_app_load()
+
+    #
+    # open Contacts
+    #
+    d.start("Open contacts app")
+    contacts = nc.open_contacts()
+    nc.wait_for_app_load()
+
+    #
+    # handle selected operation 
+    #
+    if op=='exists':
+        d.start("Check that contact %s exists", contact['email'])
+        contacts.click_contact(contact) # raises NoSuchElementException if not found
+        
+    elif op=='delete':
+        d.start("Delete contact %s", contact['email'])
+        contacts.click_contact(contact)
+        contacts.wait_contact_loaded()
+        contacts.delete_current_contact()
+        
+    else:
+        raise ValueError('Invalid operation: %s' % op)
+
+    #
+    # logout
+    #
+    d.start("Logout")
+    nc.logout()
+    nc.wait_for_login_screen()
+
+    #
+    # done
+    #
+    d.start("Success!")
+
+except Exception as e:
+    d.fail(e)
+    raise
+
+finally:
+    d.quit()

tests/suites/remote-nextcloud.sh

@@ -127,12 +127,6 @@ test_nextcloud_contacts() {
     carddav_delete_contact "$alice" "$alice_pw" "$c_uid" 2>>$TEST_OF || \
         test_failure "Unable to delete contact for $alice in Nextcloud"
     
-
-    #
-    # 2. create contact in Roundcube - ensure contact appears in Nextcloud
-    #
-    # TODO
-
     
     # clean up
     mgmt_assert_delete_user "$alice"

tests/suites/roundcube.sh

@@ -0,0 +1,127 @@
+#####
+##### This file is part of Mail-in-a-Box-LDAP which is released under the
+##### terms of the GNU Affero General Public License as published by the
+##### Free Software Foundation, either version 3 of the License, or (at
+##### your option) any later version. See file LICENSE or go to
+##### https://github.com/downtownallday/mailinabox-ldap for full license
+##### details.
+#####
+
+
+test_password_change() {
+    # ensure user passwords can be changed from roundcube
+    test_start "password-change"
+
+    # create regular user alice
+    local alice="alice@somedomain.com"
+    local alice_old_pw="alice_1234"
+    local alice_new_pw="123_new_alice"
+    create_user "$alice" "$alice_old_pw"
+
+    # change the password using roundcube's ui
+    assert_browser_test \
+        roundcube/change_pw.py \
+        "$alice" \
+        "$alice_old_pw" \
+        "$alice_new_pw"
+
+    # test login using new password
+    if ! have_test_failures; then
+        get_attribute "$LDAP_USERS_BASE" "mail=$alice" "dn"
+        assert_r_access "$ATTR_DN" "$ATTR_DN" "$alice_new_pw" read mail
+    fi
+    
+    # clean up
+    delete_user "$alice"
+    test_end
+}
+
+
+test_create_contact() {
+    #
+    # ensure contacts can be created in Roundcube and that those
+    # contacts appears in Nextcloud (support both local and remote
+    # Nextcloud setups)
+    #
+    # we're not going to check that a contact created in Nextcloud is
+    # available in Roundcube because there's already a test for that
+    # in the suite remote-nextcloud.sh
+    #
+    test_start "create-contact"
+
+    # create regular user alice
+    local alice="alice@somedomain.com"
+    local alice_pw="$(generate_password 16)"
+    create_user "$alice" "$alice_pw"
+
+    # which address book in roundcube?
+    # .. local nextcloud: the name is "ownCloud (Contacts)
+    # .. remote nextcloud: the name is the remote server name
+    #
+    # RCM_PLUGIN_DIR is defined in lib/locations.sh    
+    record "[get address book name]"
+    local code address_book
+    address_book=$(php${PHP_VER} -r "require '$RCM_PLUGIN_DIR/carddav/config.inc.php'; isset(\$prefs['cloud']) ? print \$prefs['cloud']['name'] : print \$prefs['ownCloud']['name'];" 2>>$TEST_OF)
+    record "name: $address_book"
+    code=$?
+    if [ $code -ne 0 ]; then
+        test_failure "Could not determine the address book name to use"
+
+    else
+        # generate an email address - the contact's email must be
+        # unique or it can't be created
+        local contact_email="bob_bacon$(generate_uuid | awk -F- '{print $1 }')@example.com"
+        
+        # create a contact using roundcube's ui
+        record "[create contact in Roundcube]"
+        if assert_browser_test \
+               roundcube/create_contact.py \
+               "$alice" \
+               "$alice_pw" \
+               "$address_book" \
+               "Bob" \
+               "Bacon" \
+               "$contact_email"
+        then
+            # ensure the contact exists in Nextcloud.
+            #
+            # skip explicitly checking for existance - when we delete
+            # the contact we're also checking that it exists (delete
+            # will fail otherwise)
+            
+            # record "[ensure contact exists in Nextcloud]"
+            # assert_browser_test \
+            #     nextcloud/contacts.py \
+            #     "exists" \
+            #     "$alice" \
+            #     "$alice_pw" \
+            #     "Bob" \
+            #     "Bacon" \
+            #     "$contact_email"
+
+            # delete the contact
+            record "[delete the contact in Nextcloud]"
+            assert_browser_test \
+                nextcloud/contacts.py \
+                "delete" \
+                "$alice" \
+                "$alice_pw" \
+                "Bob" \
+                "Bacon" \
+                "$contact_email"
+        fi
+    fi
+    
+    delete_user "$alice"
+    test_end
+    
+}
+
+
+
+suite_start "roundcube"
+
+test_password_change
+test_create_contact
+
+suite_end

tests/suites/roundcube/change_pw.py

@@ -0,0 +1,70 @@
+#####
+##### This file is part of Mail-in-a-Box-LDAP which is released under the
+##### terms of the GNU Affero General Public License as published by the
+##### Free Software Foundation, either version 3 of the License, or (at
+##### your option) any later version. See file LICENSE or go to
+##### https://github.com/downtownallday/mailinabox-ldap for full license
+##### details.
+#####
+
+from browser.automation import (
+    TestDriver,
+    TimeoutException,
+    NoSuchElementException
+)
+from browser.RoundcubeAutomation import RoundcubeAutomation
+import sys
+
+login = sys.argv[1]
+old_pw = sys.argv[2]
+new_pw = sys.argv[3]
+
+d = TestDriver()
+rcm = RoundcubeAutomation(d)
+
+try:
+    #
+    # open the browser to roundcube
+    #
+    d.start("Opening roundcube")
+    d.get("/mail/")
+    rcm.wait_for_login_screen(secs=10)
+
+    #
+    # login
+    #
+    rcm.login(login, old_pw)
+    rcm.wait_for_inbox()
+
+    #
+    # change password
+    #
+    d.start("Change password")
+    rcm.open_settings()
+    rcm.wait_for_settings()
+    
+    d.say("Enter new password")
+    d.find_el('a.password').click() # open the change password section
+    d.wait_for_el('button[value=Save]') # wait for it to load
+    d.find_el('#curpasswd').send_text(old_pw) # fill old password
+    d.find_el('#newpasswd').send_text(new_pw) # fill new password
+    d.find_el('#confpasswd').send_text(new_pw) # fill confirm password
+    d.find_el('button[value=Save]').click()    # save new password
+    d.wait_for_text("Successfully saved", secs=5, case_sensitive=False)
+
+    #
+    # logout
+    #
+    rcm.logout()
+    
+    #
+    # done
+    #
+    d.say("Success!")
+
+except Exception as e:
+    d.fail(e)
+    raise
+
+finally:
+    d.quit()

tests/suites/roundcube/create_contact.py

@@ -0,0 +1,88 @@
+#####
+##### This file is part of Mail-in-a-Box-LDAP which is released under the
+##### terms of the GNU Affero General Public License as published by the
+##### Free Software Foundation, either version 3 of the License, or (at
+##### your option) any later version. See file LICENSE or go to
+##### https://github.com/downtownallday/mailinabox-ldap for full license
+##### details.
+#####
+
+from browser.automation import (
+    TestDriver,
+    TimeoutException,
+    NoSuchElementException,
+    ElementNotInteractableException,
+)
+from browser.RoundcubeAutomation import RoundcubeAutomation
+import sys
+
+login = sys.argv[1]
+pw = sys.argv[2]
+address_book = sys.argv[3]
+contact = {
+    'givenname': sys.argv[4],
+    'surname': sys.argv[5],
+    'email': sys.argv[6],
+}
+
+d = TestDriver()
+rcm = RoundcubeAutomation(d)
+
+try:
+    #
+    # open the browser to roundcube
+    #
+    d.start("Opening roundcube")
+    d.get("/mail/")
+    rcm.wait_for_login_screen(secs=10)
+
+    #
+    # login
+    #
+    rcm.login(login, pw)
+    rcm.wait_for_inbox()
+
+    #
+    # add contact
+    #
+    d.start("Add contact")
+    rcm.open_contacts()
+    rcm.wait_for_contacts()
+
+    d.say("Select address book '%s'", address_book)
+    el = d.find_text(address_book, "a", exact=True, throws=False, case_sensitive=True)
+    if not el:
+        el = d.find_text(address_book + ' (Contacts)', "a", exact=True, case_sensitive=True)
+    if not el.is_displayed():
+        d.say_verbose("open sidebar to select address book")
+        d.find_el('a.back-sidebar-button').click()
+    el.click()
+
+    d.say("Create contact")
+    d.find_el('a.create').click()
+    iframe_el = d.wait_for_el('#contact-frame', secs=5)
+    d.switch_to_frame(iframe_el)
+    
+    d.find_el('#ff_firstname').send_text(contact['givenname'])
+    d.find_el('#ff_surname').send_text(contact['surname'])
+    d.find_el('#ff_email0').send_text(contact['email'])
+    d.find_el('button[value=Save]').click()    # save new password
+    d.switch_to_window(d.get_current_window_handle())
+    d.wait_for_text("Successfully saved", secs=5, case_sensitive=False)
+
+    #
+    # logout
+    #
+    rcm.logout()
+    
+    #
+    # done
+    #
+    d.say("Success!")
+
+except Exception as e:
+    d.fail(e)
+    raise
+
+finally:
+    d.quit()

tests/suites/run_browser_test.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+#####
+##### This file is part of Mail-in-a-Box-LDAP which is released under the
+##### terms of the GNU Affero General Public License as published by the
+##### Free Software Foundation, either version 3 of the License, or (at
+##### your option) any later version. See file LICENSE or go to
+##### https://github.com/downtownallday/mailinabox-ldap for full license
+##### details.
+#####
+
+
+# use this to run a browser test from the command line
+
+mydir=$(dirname "$0")
+export PYTHONPATH=$(realpath "$mydir/../lib/python"):$PYTHONPATH
+export BROWSER_TESTS_VERBOSITY=3
+
+python3 "$@"

tests/system-setup/setup-funcs.sh

@@ -91,6 +91,8 @@ init_test_system() {
     H2 "apt-get update"
     wait_for_apt
     exec_no_output apt-get update -qq || die "apt-get update failed!"
+    H2 "snap refresh"
+    exec_no_output snap refresh || echo "snap refresh failed! ignoring..."
 
     # install .emacs file, if available
     if [ -e tests/assets/.emacs -a -d /root ]; then
@@ -150,15 +152,22 @@ init_miab_testing() {
     
     H2 "QA prerequisites"
     local rc=0
-    
+
+    # python3-pip: installed by setup, but we need it now
     # python3-dnspython: is used by the python scripts in 'tests' and is
     #   not installed by setup
     # also install 'jq' for json processing
-    echo "Install python3-dnspython, jq, git"
+    echo "Install python3-pip, python3-dnspython, jq, git"
     wait_for_apt
-    exec_no_output apt-get install -y python3-dnspython jq git \
+    exec_no_output apt-get install -y python3-pip python3-dnspython jq git \
         || die "Unable to install setup prerequisites !!"
 
+    # browser-based tests
+    echo "Install chromium, selenium"
+    exec_no_output snap install chromium \
+        || die "Unable to install chromium!"
+    exec_no_output python3 -m pip install selenium --quiet \
+        || die "Selenium install failed!"
 
     # tell git our directory is safe (new requirement for git 2.35.2)
     if [ -d .git ]; then

tests/vagrant/preloaded/prepvm.sh

@@ -32,6 +32,7 @@ source tests/lib/system.sh
 source tests/lib/color-output.sh
 
 dry_run=true
+start=$(date +%s)
 
 if [ "$1" == "--no-dry-run" ]; then
     dry_run=false
@@ -135,9 +136,9 @@ PHP_VER=$(source setup/functions.sh; echo $PHP_VER)
 if ! $dry_run; then
     H1 "Upgrade system"
     H2 "apt update"
-    exec_no_output apt-get update -y
+    exec_no_output apt-get update -y || exit 1
     H2 "apt upgrade"
-    exec_no_output apt-get upgrade -y --with-new-pkgs
+    exec_no_output apt-get upgrade -y --with-new-pkgs || exit 1
     H2 "apt autoremove"
     exec_no_output apt-get autoremove -y
 fi
@@ -179,38 +180,52 @@ for file in "${desired_order[@]}" "${setup_files[@]}"; do
     fi
 done
 
+failed=0
+    
 for file in ${ordered_files[@]}; do
     H1 "$file"
     remove_line_continuation "$file" | install_packages
-    [ $? -ne 0 ] && exit 1
+    [ $? -ne 0 ] && let failed+=1
 done
 
 if ! $dry_run; then
     # bonus
     H1 "install extras"
-    H2 "openssh-server"
-    exec_no_output apt-get install -y openssh-server
-    # ssh-rsa no longer a default algorithm, but still used by vagrant
-    # echo "PubkeyAcceptedAlgorithms +ssh-rsa" > /etc/ssh/sshd_config.d/miabldap.conf
-    H2 "emacs"
-    exec_no_output apt-get install -y emacs-nox
-    H2 "ntpdate"
-    exec_no_output apt-get install -y ntpdate
-    H2 "net-tools"
-    exec_no_output apt-get install -y net-tools
+
+    H2 "openssh, emacs, ntpdate, net-tools, jq"
+    exec_no_output apt-get install -y openssh-server emacs-nox ntpdate net-tools jq || let failed+=1
 
     # these are added by system-setup scripts and needed for test runner
     H2 "python3-dnspython"
-    exec_no_output apt-get install -y python3-dnspython
-    H2 "jq"
-    exec_no_output apt-get install -y jq
+    exec_no_output apt-get install -y python3-dnspython || let failed+=1
+    H2 "pyotp(pip)"
+    exec_no_output python3 -m pip install pyotp --quiet || let failed+=1
+
+    # ...and for browser-based tests
+    #H2 "x11"  # needed for chromium w/head (not --headless)
+    #exec_no_output apt-get install -y xorg openbox xvfb gtk2-engines-pixbuf dbus-x11 xfonts-base xfonts-100dpi xfonts-75dpi xfonts-cyrillic xfonts-scalable x11-apps imagemagick || let failed+=1
+    H2 "chromium"
+    #exec_no_output apt-get install -y chromium-browser || let failed+=1
+    exec_no_output snap install chromium || let failed+=1
+    H2 "selenium(pip)"
+    exec_no_output python3 -m pip install selenium --quiet || let failed+=1
 
     # remove apache, which is what setup will do
     H2 "remove apache2"
     exec_no_output apt-get -y purge apache2 apache2-\*
 
-    echo ""
-    echo ""
-    echo "Done. Take a snapshot...."
-    echo ""
 fi
+
+end=$(date +%s)
+echo ""
+echo ""
+if [ $failed -gt 0 ]; then
+    echo "$failed failures! ($(elapsed_pretty $start $end))"
+    echo ""
+    exit 1
+else
+    echo "Successfully prepped in $(elapsed_pretty $start $end). Take a snapshot...."
+    echo ""
+    exit 0
+fi
+