add routes test

ckan_default.conf

+<VirtualHost *:80>
+    WSGIScriptAlias / /etc/ckan/default/apache.wsgi
+
+    # Pass authorization info on (needed for rest api).
+    WSGIPassAuthorization On
+
+    # Deploy as a daemon (avoids conflicts between CKAN instances).
+    WSGIDaemonProcess ckan_default display-name=ckan_default processes=2 threads=15
+
+    WSGIProcessGroup ckan_default
+    RewriteEngine On
+
+    #resources
+    RewriteCond %{REQUEST_URI} !^/.*.css
+    RewriteCond %{REQUEST_URI} !^/base/images/
+    RewriteCond %{REQUEST_URI} !^/base/fonts/
+    RewriteCond %{REQUEST_URI} !^/fanstatic/
+    RewriteCond %{REQUEST_URI} !^/scripts/vendor/
+    RewriteCond %{REQUEST_URI} !^/uploads/group/
+    #api
+    RewriteCond %{REQUEST_URI} !^/api/i18n/de$
+    RewriteCond %{REQUEST_URI} !^/api/2/.*$
+    RewriteCond %{REQUEST_URI} !^/api/3/action/package.*$
+    RewriteCond %{REQUEST_URI} !^/catalog.xml[^/]*$
+    #user       
+    RewriteCond %{REQUEST_URI} !^/user/login$
+    RewriteCond %{REQUEST_URI} !^/user/logged_out_redirect$
+    RewriteCond %{REQUEST_URI} !^/user/reset$
+    RewriteCond %{REQUEST_URI} !^/user/edit$
+    RewriteCond %{REQUEST_URI} !^/user/register$
+    RewriteCond %{REQUEST_URI} !^/user/\w+$
+    RewriteCond %{REQUEST_URI} !^/user/$
+    RewriteCond %{REQUEST_URI} !^/login_generic?
+    RewriteCond %{REQUEST_URI} !^/logged_in?
+    #report
+    RewriteCond %{REQUEST_URI} !^/report$
+    RewriteCond %{REQUEST_URI} !^/report/openness$
+    RewriteCond %{REQUEST_URI} !^/report/openness/[^/]*$
+    #organization
+    RewriteCond %{REQUEST_URI} !^/organization$
+    RewriteCond %{REQUEST_URI} !^/organization/new$
+    RewriteCond %{REQUEST_URI} !^/organization?__no_cache__=True$
+    RewriteCond %{REQUEST_URI} !^/organization/[^/]*$
+    RewriteCond %{REQUEST_URI} !^/organization/edit/[^/]*$
+    RewriteCond %{REQUEST_URI} !^/organization/delete/[^/]*$
+    RewriteCond %{REQUEST_URI} !^/organization/members/[^/]*$
+    RewriteCond %{REQUEST_URI} !^/organization/member_new/[^/]*$
+    RewriteCond %{REQUEST_URI} !^/organization/member_delete/[^/]*$
+    #dataset
+    RewriteCond %{REQUEST_URI} !^/dataset$
+    RewriteCond %{REQUEST_URI} !^/dataset/[^/]*$
+    RewriteCond %{REQUEST_URI} !^/dataset/new_resource/[^/]*$
+    RewriteCond %{REQUEST_URI} !^/dataset/edit/[^/]*$
+    RewriteCond %{REQUEST_URI} !^/dataset/[^/]+/resource/[^/]+$
+    RewriteCond %{REQUEST_URI} !^/dataset/[^/]+/resource_edit/[^/]*$
+    RewriteCond %{REQUEST_URI} !^/dataset/[^/]+/resource_data/[^/]+$
+    RewriteCond %{REQUEST_URI} !^/dataset/[^/]+/resource_delete/[^/]*$
+    RewriteCond %{REQUEST_URI} !^/dataset/[^/]+/resource/[^/]+/download/[^/]+$
+    RewriteCond %{REQUEST_URI} !^/dataset/[^/]+/resource/[^/]+/edit_view/[^/]+$
+    RewriteCond %{REQUEST_URI} !^/dataset/delete/[^/]+$
+    #tag
+    RewriteCond %{REQUEST_URI} !^/tag/[^/]*$
+    #harvest
+    RewriteCond %{REQUEST_URI} !^/harvest.*$
+    #feed
+    RewriteCond %{REQUEST_URI} !^/feeds/custom.atom[^/]*$
+    #other
+    RewriteCond %{REQUEST_URI} !^/$
+    RewriteCond %{REQUEST_URI} !^/info_page$
+    #
+    RewriteCond %{REQUEST_URI} !^/notfound$
+    #block if no match
+    RewriteRule (.*) /notfound [P,L,NE]
+
+    RewriteCond %{REQUEST_URI} ^/user/dashboard [OR]
+    RewriteCond %{REQUEST_URI} ^/user/me
+    RewriteRule (.*) /notfound [P,L,NE]
+
+
+    ErrorLog /var/log/apache2/ckan_default.error.log
+    CustomLog /var/log/apache2/ckan_default.custom.log combined
+LogLevel alert rewrite:trace3 alias:debug
+
+    <IfModule mod_rpaf.c>
+        RPAFenable On
+        RPAFsethostname On
+        RPAFproxy_ips 127.0.0.1
+    </IfModule>
+
+    <Directory />
+        Require all granted
+    </Directory>
+
+    # ProxyPreserveHost On
+
+    # ProxyPass /dataset/new http://10.61.47.219/dataset/new
+    # ProxyPassReverse /dataset/new http://10.61.47.219/dataset/new
+    # ProxyPassMatch ^/(dataset/delete/[^/]+)$ http://10.61.47.219/$1
+    # ProxyPassReverse ^/(dataset/delete/[^/]+)$ http://10.61.47.219/$1
+    # ProxyPassMatch ^/(dataset/edit/[^/]+)$ http://10.61.47.219/$1
+    # ProxyPassReverse ^/(dataset/edit/[^/]+)$ http://10.61.47.219/$1
+    # ProxyPassReverse /dataset http://141.91.184.90/dataset
+    # ProxyPassReverse /dataset http://141.91.184.90/dataset
+    # ProxyPass /solr http://localhost:8983/solr
+    # ProxyPassReverse /solr http://localhost:8983/solr
+    # ProxyPass /dataset/new_resource http://10.61.47.219/dataset/new_resource
+    # ProxyPassReverse /dataset/new_resource http://141.91.184.90/dataset/new_resource
+    # ProxyPassReverse /dataset/new_resource http://141.91.184.90/dataset/new_resource
+    # #ProxyPass /api/i18n/de http://141.91.184.90/api/i18n/de
+    # ProxyPassReverse ^/uploads/group/(.*)$ http://10.61.47.219/uploads/group/$1
+    # ProxyPassMatch ^/uploads/group/(.*)$ http://10.61.47.219/uploads/group/$1
+    # ProxyPassReverse ^/(dataset/[^/]+/resource/[^/]+/download/[^/]+)$ http://141.91.184.90/$1
+    # ProxyPassMatch ^/(dataset/[^/]+/resource/[^/]+/download/[^/]+)$ http://141.91.184.90/$1
+    # ProxyPassReverse ^/(dataset/[^/]+/resource/[^/]+)$ http://10.61.47.219/$1
+    # ProxyPassMatch ^/(dataset/[^/]+/resource/[^/]+/)$ http://10.61.47.219/$1
+    # ProxyPassMatch ^/(dataset/[^/]+/resource_data/[^/]+)$ http://10.61.47.219/$1
+    # ProxyPassReverse ^/(dataset/[^/]+/resource_data/[^/]+)$ http://10.61.47.219/$1
+    # ProxyPassMatch ^/(dataset/[^/]+/resource/[^/]+/new_view[^/]*)$ http://10.61.47.219/$1
+    # ProxyPassReverse ^/(dataset/[^/]+/resource/[^/]+/new_view[^/]*)$ http://10.61.47.219/$1
+    # ProxyPassMatch ^/(harvest.*)$ http://141.91.184.90/$1
+    # ProxyPassReverse /harvest http://141.91.184.90/harvest
+    # ProxyPass /harvest http://141.91.184.90/harvest
+    # ProxyPassReverse ^/(harvest.*)$ http://141.91.184.90/$1
+    # ProxyPassReverse /harvest/admin http://141.91.184.90/harvest/admin
+    # ProxyPassReverse ^/(api/3/action/package.*)$ http://10.61.47.219/$1
+    # ProxyPassMatch ^/(api/3/action/package.*)$ http://10.61.47.219/$1
+    # ProxyPass /api/action/package_create http://10.61.47.219/api/action/package_create
+    # ProxyPassReverse /api/action/package_create http://10.61.47.219/api/action/package_create
+    # ProxyPass /api/action/resource_create http://10.61.47.219/api/action/resource_create
+    # ProxyPassReverse /api/action/resource_create http://10.61.47.219/api/action/resource_create
+    # ProxyPassMatch ^/(organization/edit/[^/]+)$ http://10.61.47.219/$1
+    # ProxyPassReverse ^/(organization/edit/[^/]+)$ http://10.61.47.219/$1 
+    # ProxyPassReverse /organization http://141.91.184.90/organization
+    # ProxyPassMatch ^/(organization/delete/[^/]+)$ http://10.61.47.219/$1
+    # ProxyPass /datarequest http://10.61.47.219/datarequest
+    # ProxyPassReverse /datarequest http://10.61.47.219/datarequest
+
+       ProxyPass /dataset/new http://<master-ip>/dataset/new
+   ProxyPassReverse /dataset/new http://<master-ip>/dataset/new
+   ProxyPassMatch ^/(dataset/delete/[^/]+)$ http://<master-ip>/$1
+   ProxyPassReverse ^/(dataset/delete/[^/]+)$ http://<master-ip>/$1
+   ProxyPassMatch ^/(dataset/edit/[^/]+)$ http://<master-ip>/$1
+   ProxyPassReverse ^/(dataset/edit/[^/]+)$ http://<master-ip>/$1
+   ProxyPassReverse /dataset http://<master-ip>/dataset
+   ProxyPassReverse /dataset http://<master-ip>/dataset
+   ProxyPass /dataset/new_resource http://<master-ip>/dataset/new_resource
+   ProxyPassReverse /dataset/new_resource http://<master-ip>/dataset/new_resource
+   ProxyPassReverse /dataset/new_resource http://<master-ip>/dataset/new_resource
+   ProxyPassReverse ^/uploads/group/(.*)$ http://<master-ip>/uploads/group/$1
+   ProxyPassMatch ^/uploads/group/(.*)$ http://<master-ip>/uploads/group/$1
+   ProxyPassReverse ^/(dataset/[^/]+/resource/[^/]+/download/[^/]+)$ http://<master-ip>/$1
+   ProxyPassMatch ^/(dataset/[^/]+/resource/[^/]+/download/[^/]+)$ http://<master-ip>/$1
+   ProxyPassReverse ^/(dataset/[^/]+/resource/[^/]+)$ http://<master-ip>/$1
+   ProxyPassMatch ^/(dataset/[^/]+/resource/[^/]+/)$ http://<master-ip>/$1
+   ProxyPassMatch ^/(dataset/[^/]+/resource_data/[^/]+)$ http://<master-ip>/$1
+   ProxyPassReverse ^/(dataset/[^/]+/resource_data/[^/]+)$ http://<master-ip>/$1
+   ProxyPassMatch ^/(dataset/[^/]+/resource/[^/]+/new_view[^/]*)$ http://<master-ip>/$1
+   ProxyPassReverse ^/(dataset/[^/]+/resource/[^/]+/new_view[^/]*)$ http://<master-ip>/$1
+   ProxyPassReverse ^/(harvest.*)$ http://<master-ip>/$1
+   ProxyPassMatch ^/(harvest.*)$ http://<master-ip>/$1
+   ProxyPassReverse ^/(api/3/action/package.*)$ http://<master-ip>/$1
+   ProxyPassMatch ^/(api/3/action/package.*)$ http://<master-ip>/$1
+   ProxyPassMatch ^/(organization/edit/[^/]+)$ http://<master-ip>/$1
+   ProxyPassReverse ^/(organization/edit/[^/]+)$ http://<master-ip>/$1
+   ProxyPass /organization/new http://<interne-IP-Master>/organization/new
+ProxyPassReverse /organization/new http://<interne-IP-Master>/organization/new
+</VirtualHost
\ No newline at end of file

ckanext/odsh/tests/test_env.py

@@ -8,6 +8,7 @@ import os
 import sys
 import ConfigParser
 from collections import OrderedDict
+from urlparse import urlsplit
 
 expected_commit = '8cd9576884cae6abe50a27c891434cb9fe87ced2'
 
@@ -178,3 +179,91 @@ class TestEnv:
     #     version = readUrl(url+'/api/3/action/resource_qv4yAI2rgotamXGk98gJ').strip()
     #     # version = checkConfig('ckanext.odsh.version')
     #     assert version == expected_commit, "wrong version: {was}!={exp}".format(was=version, exp=expected_commit)
+
+    def test_routes(self):
+        # if isMaster():
+        #     return
+
+        expexted_rules = \
+            """ ProxyPass /dataset/new http://10.61.47.219/dataset/new
+    ProxyPassReverse /dataset/new http://10.61.47.219/dataset/new
+    ProxyPassMatch ^/(dataset/delete/[^/]+)$ http://10.61.47.219/$1
+    ProxyPassReverse ^/(dataset/delete/[^/]+)$ http://10.61.47.219/$1
+    ProxyPassMatch ^/(dataset/edit/[^/]+)$ http://10.61.47.219/$1
+    ProxyPassReverse ^/(dataset/edit/[^/]+)$ http://10.61.47.219/$1
+    ProxyPassReverse /dataset http://141.91.184.90/dataset
+    ProxyPassReverse /dataset http://141.91.184.90/dataset
+    ProxyPass /dataset/new_resource http://10.61.47.219/dataset/new_resource
+    ProxyPassReverse /dataset/new_resource http://141.91.184.90/dataset/new_resource
+    ProxyPassReverse /dataset/new_resource http://141.91.184.90/dataset/new_resource
+    #ProxyPass /api/i18n/de http://141.91.184.90/api/i18n/de
+    ProxyPassReverse ^/uploads/group/(.*)$ http://10.61.47.219/uploads/group/$1
+    ProxyPassMatch ^/uploads/group/(.*)$ http://10.61.47.219/uploads/group/$1
+    ProxyPassReverse ^/(dataset/[^/]+/resource/[^/]+/download/[^/]+)$ http://141.91.184.90/$1
+    ProxyPassMatch ^/(dataset/[^/]+/resource/[^/]+/download/[^/]+)$ http://141.91.184.90/$1
+    ProxyPassReverse ^/(dataset/[^/]+/resource/[^/]+)$ http://10.61.47.219/$1
+    ProxyPassMatch ^/(dataset/[^/]+/resource/[^/]+/)$ http://10.61.47.219/$1
+    ProxyPassMatch ^/(dataset/[^/]+/resource_data/[^/]+)$ http://10.61.47.219/$1
+    ProxyPassReverse ^/(dataset/[^/]+/resource_data/[^/]+)$ http://10.61.47.219/$1
+    ProxyPassMatch ^/(dataset/[^/]+/resource/[^/]+/new_view[^/]*)$ http://10.61.47.219/$1
+    ProxyPassReverse ^/(dataset/[^/]+/resource/[^/]+/new_view[^/]*)$ http://10.61.47.219/$1
+    ProxyPassMatch ^/(harvest.*)$ http://141.91.184.90/$1
+    ProxyPassReverse /harvest http://141.91.184.90/harvest
+    ProxyPass /harvest http://141.91.184.90/harvest
+    ProxyPassReverse ^/(harvest.*)$ http://141.91.184.90/$1
+    ProxyPassReverse ^/(api/3/action/package.*)$ http://10.61.47.219/$1
+    ProxyPassMatch ^/(api/3/action/package.*)$ http://10.61.47.219/$1
+    ProxyPass /api/action/package_create http://10.61.47.219/api/action/package_create
+    ProxyPassReverse /api/action/package_create http://10.61.47.219/api/action/package_create
+    ProxyPass /api/action/resource_create http://10.61.47.219/api/action/resource_create
+    ProxyPassReverse /api/action/resource_create http://10.61.47.219/api/action/resource_create
+    ProxyPassMatch ^/(organization/edit/[^/]+)$ http://10.61.47.219/$1
+    ProxyPassReverse ^/(organization/edit/[^/]+)$ http://10.61.47.219/$1 
+    ProxyPass /organization/new http://<interne-IP-Master>/organization/new
+    ProxyPassReverse /organization/new http://<interne-IP-Master>/organization/new
+    ProxyPassReverse /organization http://<interne-IP-Master>/organization
+    ProxyPassReverse ^/(organization/edit/[^/]+)$ http://<interne-IP-Master>/$1
+
+    # ProxyPass /datarequest http://10.61.47.219/datarequest
+    # ProxyPassReverse /datarequest http://10.61.47.219/datarequest
+    """
+
+        expected = self._parse_rules(expexted_rules.splitlines())
+
+        # with open('ckan_default.conf', 'r') as aconfig:
+        with open('/etc/apache2/sites-enabled/ckan_default.conf', 'r') as aconfig:
+            lines = aconfig.readlines()
+            # pdb.set_trace()
+            current = self._parse_rules(lines, check_host=True)
+            if len(expected.symmetric_difference(current)) > 0:
+                diff = expected.difference(current)
+                if len(diff) > 0:
+                    print('WARNING: missing routes:')
+                    for r in sorted(diff, key=lambda tup: tup[1]):
+                        print('{cmd} {source} {target}'.format(
+                            cmd=r[0], source=r[1], target='http://<interne-IP-Master>'+r[2]))
+                diff = current.difference(expected)
+                if len(diff) > 0:
+                    print('WARNING: found unexpected routes:')
+                    for r in sorted(diff, key=lambda tup: tup[1]):
+                        print('{cmd} {source} {target}'.format(
+                            cmd=r[0], source=r[1], target='<target>'+r[2]))
+
+    def _parse_rules(self, lines, check_host=False):
+        rules = set(['ProxyPassMatch', 'ProxyPassReverse', 'ProxyPass'])
+        ret = []
+        hosts = set()
+        for line in lines:
+            tokens = filter(lambda t: t.strip(), line.strip().split(' '))
+            if not tokens or tokens[0] not in rules:
+                continue
+            assert len(tokens) == 3
+            # for token in tokens:
+            # print(token)
+            f = urlsplit(tokens[2])
+            ret.append((tokens[0], tokens[1], f.path))
+            hosts.add(f.netloc)
+        if check_host and len(hosts) > 1:
+            print('WARNING: found multiple target hosts: {hosts}'.format(
+                hosts=', '.join(hosts)))
+        return set(ret)