diff --git a/ckan_default.conf b/ckan_default.conf new file mode 100644 index 0000000000000000000000000000000000000000..891eaf22b4291dc46b74b6b2ba988b5e9703a65e --- /dev/null +++ b/ckan_default.conf @@ -0,0 +1,167 @@ +<VirtualHost *:80> + WSGIScriptAlias / /etc/ckan/default/apache.wsgi + + # Pass authorization info on (needed for rest api). + WSGIPassAuthorization On + + # Deploy as a daemon (avoids conflicts between CKAN instances). + WSGIDaemonProcess ckan_default display-name=ckan_default processes=2 threads=15 + + WSGIProcessGroup ckan_default + RewriteEngine On + + #resources + RewriteCond %{REQUEST_URI} !^/.*.css + RewriteCond %{REQUEST_URI} !^/base/images/ + RewriteCond %{REQUEST_URI} !^/base/fonts/ + RewriteCond %{REQUEST_URI} !^/fanstatic/ + RewriteCond %{REQUEST_URI} !^/scripts/vendor/ + RewriteCond %{REQUEST_URI} !^/uploads/group/ + #api + RewriteCond %{REQUEST_URI} !^/api/i18n/de$ + RewriteCond %{REQUEST_URI} !^/api/2/.*$ + RewriteCond %{REQUEST_URI} !^/api/3/action/package.*$ + RewriteCond %{REQUEST_URI} !^/catalog.xml[^/]*$ + #user + RewriteCond %{REQUEST_URI} !^/user/login$ + RewriteCond %{REQUEST_URI} !^/user/logged_out_redirect$ + RewriteCond %{REQUEST_URI} !^/user/reset$ + RewriteCond %{REQUEST_URI} !^/user/edit$ + RewriteCond %{REQUEST_URI} !^/user/register$ + RewriteCond %{REQUEST_URI} !^/user/\w+$ + RewriteCond %{REQUEST_URI} !^/user/$ + RewriteCond %{REQUEST_URI} !^/login_generic? + RewriteCond %{REQUEST_URI} !^/logged_in? + #report + RewriteCond %{REQUEST_URI} !^/report$ + RewriteCond %{REQUEST_URI} !^/report/openness$ + RewriteCond %{REQUEST_URI} !^/report/openness/[^/]*$ + #organization + RewriteCond %{REQUEST_URI} !^/organization$ + RewriteCond %{REQUEST_URI} !^/organization/new$ + RewriteCond %{REQUEST_URI} !^/organization?__no_cache__=True$ + RewriteCond %{REQUEST_URI} !^/organization/[^/]*$ + RewriteCond %{REQUEST_URI} !^/organization/edit/[^/]*$ + RewriteCond %{REQUEST_URI} !^/organization/delete/[^/]*$ + RewriteCond %{REQUEST_URI} !^/organization/members/[^/]*$ + RewriteCond %{REQUEST_URI} !^/organization/member_new/[^/]*$ + RewriteCond %{REQUEST_URI} !^/organization/member_delete/[^/]*$ + #dataset + RewriteCond %{REQUEST_URI} !^/dataset$ + RewriteCond %{REQUEST_URI} !^/dataset/[^/]*$ + RewriteCond %{REQUEST_URI} !^/dataset/new_resource/[^/]*$ + RewriteCond %{REQUEST_URI} !^/dataset/edit/[^/]*$ + RewriteCond %{REQUEST_URI} !^/dataset/[^/]+/resource/[^/]+$ + RewriteCond %{REQUEST_URI} !^/dataset/[^/]+/resource_edit/[^/]*$ + RewriteCond %{REQUEST_URI} !^/dataset/[^/]+/resource_data/[^/]+$ + RewriteCond %{REQUEST_URI} !^/dataset/[^/]+/resource_delete/[^/]*$ + RewriteCond %{REQUEST_URI} !^/dataset/[^/]+/resource/[^/]+/download/[^/]+$ + RewriteCond %{REQUEST_URI} !^/dataset/[^/]+/resource/[^/]+/edit_view/[^/]+$ + RewriteCond %{REQUEST_URI} !^/dataset/delete/[^/]+$ + #tag + RewriteCond %{REQUEST_URI} !^/tag/[^/]*$ + #harvest + RewriteCond %{REQUEST_URI} !^/harvest.*$ + #feed + RewriteCond %{REQUEST_URI} !^/feeds/custom.atom[^/]*$ + #other + RewriteCond %{REQUEST_URI} !^/$ + RewriteCond %{REQUEST_URI} !^/info_page$ + # + RewriteCond %{REQUEST_URI} !^/notfound$ + #block if no match + RewriteRule (.*) /notfound [P,L,NE] + + RewriteCond %{REQUEST_URI} ^/user/dashboard [OR] + RewriteCond %{REQUEST_URI} ^/user/me + RewriteRule (.*) /notfound [P,L,NE] + + + ErrorLog /var/log/apache2/ckan_default.error.log + CustomLog /var/log/apache2/ckan_default.custom.log combined +LogLevel alert rewrite:trace3 alias:debug + + <IfModule mod_rpaf.c> + RPAFenable On + RPAFsethostname On + RPAFproxy_ips 127.0.0.1 + </IfModule> + + <Directory /> + Require all granted + </Directory> + + # ProxyPreserveHost On + + # ProxyPass /dataset/new http://10.61.47.219/dataset/new + # ProxyPassReverse /dataset/new http://10.61.47.219/dataset/new + # ProxyPassMatch ^/(dataset/delete/[^/]+)$ http://10.61.47.219/$1 + # ProxyPassReverse ^/(dataset/delete/[^/]+)$ http://10.61.47.219/$1 + # ProxyPassMatch ^/(dataset/edit/[^/]+)$ http://10.61.47.219/$1 + # ProxyPassReverse ^/(dataset/edit/[^/]+)$ http://10.61.47.219/$1 + # ProxyPassReverse /dataset http://141.91.184.90/dataset + # ProxyPassReverse /dataset http://141.91.184.90/dataset + # ProxyPass /solr http://localhost:8983/solr + # ProxyPassReverse /solr http://localhost:8983/solr + # ProxyPass /dataset/new_resource http://10.61.47.219/dataset/new_resource + # ProxyPassReverse /dataset/new_resource http://141.91.184.90/dataset/new_resource + # ProxyPassReverse /dataset/new_resource http://141.91.184.90/dataset/new_resource + # #ProxyPass /api/i18n/de http://141.91.184.90/api/i18n/de + # ProxyPassReverse ^/uploads/group/(.*)$ http://10.61.47.219/uploads/group/$1 + # ProxyPassMatch ^/uploads/group/(.*)$ http://10.61.47.219/uploads/group/$1 + # ProxyPassReverse ^/(dataset/[^/]+/resource/[^/]+/download/[^/]+)$ http://141.91.184.90/$1 + # ProxyPassMatch ^/(dataset/[^/]+/resource/[^/]+/download/[^/]+)$ http://141.91.184.90/$1 + # ProxyPassReverse ^/(dataset/[^/]+/resource/[^/]+)$ http://10.61.47.219/$1 + # ProxyPassMatch ^/(dataset/[^/]+/resource/[^/]+/)$ http://10.61.47.219/$1 + # ProxyPassMatch ^/(dataset/[^/]+/resource_data/[^/]+)$ http://10.61.47.219/$1 + # ProxyPassReverse ^/(dataset/[^/]+/resource_data/[^/]+)$ http://10.61.47.219/$1 + # ProxyPassMatch ^/(dataset/[^/]+/resource/[^/]+/new_view[^/]*)$ http://10.61.47.219/$1 + # ProxyPassReverse ^/(dataset/[^/]+/resource/[^/]+/new_view[^/]*)$ http://10.61.47.219/$1 + # ProxyPassMatch ^/(harvest.*)$ http://141.91.184.90/$1 + # ProxyPassReverse /harvest http://141.91.184.90/harvest + # ProxyPass /harvest http://141.91.184.90/harvest + # ProxyPassReverse ^/(harvest.*)$ http://141.91.184.90/$1 + # ProxyPassReverse /harvest/admin http://141.91.184.90/harvest/admin + # ProxyPassReverse ^/(api/3/action/package.*)$ http://10.61.47.219/$1 + # ProxyPassMatch ^/(api/3/action/package.*)$ http://10.61.47.219/$1 + # ProxyPass /api/action/package_create http://10.61.47.219/api/action/package_create + # ProxyPassReverse /api/action/package_create http://10.61.47.219/api/action/package_create + # ProxyPass /api/action/resource_create http://10.61.47.219/api/action/resource_create + # ProxyPassReverse /api/action/resource_create http://10.61.47.219/api/action/resource_create + # ProxyPassMatch ^/(organization/edit/[^/]+)$ http://10.61.47.219/$1 + # ProxyPassReverse ^/(organization/edit/[^/]+)$ http://10.61.47.219/$1 + # ProxyPassReverse /organization http://141.91.184.90/organization + # ProxyPassMatch ^/(organization/delete/[^/]+)$ http://10.61.47.219/$1 + # ProxyPass /datarequest http://10.61.47.219/datarequest + # ProxyPassReverse /datarequest http://10.61.47.219/datarequest + + ProxyPass /dataset/new http://<master-ip>/dataset/new + ProxyPassReverse /dataset/new http://<master-ip>/dataset/new + ProxyPassMatch ^/(dataset/delete/[^/]+)$ http://<master-ip>/$1 + ProxyPassReverse ^/(dataset/delete/[^/]+)$ http://<master-ip>/$1 + ProxyPassMatch ^/(dataset/edit/[^/]+)$ http://<master-ip>/$1 + ProxyPassReverse ^/(dataset/edit/[^/]+)$ http://<master-ip>/$1 + ProxyPassReverse /dataset http://<master-ip>/dataset + ProxyPassReverse /dataset http://<master-ip>/dataset + ProxyPass /dataset/new_resource http://<master-ip>/dataset/new_resource + ProxyPassReverse /dataset/new_resource http://<master-ip>/dataset/new_resource + ProxyPassReverse /dataset/new_resource http://<master-ip>/dataset/new_resource + ProxyPassReverse ^/uploads/group/(.*)$ http://<master-ip>/uploads/group/$1 + ProxyPassMatch ^/uploads/group/(.*)$ http://<master-ip>/uploads/group/$1 + ProxyPassReverse ^/(dataset/[^/]+/resource/[^/]+/download/[^/]+)$ http://<master-ip>/$1 + ProxyPassMatch ^/(dataset/[^/]+/resource/[^/]+/download/[^/]+)$ http://<master-ip>/$1 + ProxyPassReverse ^/(dataset/[^/]+/resource/[^/]+)$ http://<master-ip>/$1 + ProxyPassMatch ^/(dataset/[^/]+/resource/[^/]+/)$ http://<master-ip>/$1 + ProxyPassMatch ^/(dataset/[^/]+/resource_data/[^/]+)$ http://<master-ip>/$1 + ProxyPassReverse ^/(dataset/[^/]+/resource_data/[^/]+)$ http://<master-ip>/$1 + ProxyPassMatch ^/(dataset/[^/]+/resource/[^/]+/new_view[^/]*)$ http://<master-ip>/$1 + ProxyPassReverse ^/(dataset/[^/]+/resource/[^/]+/new_view[^/]*)$ http://<master-ip>/$1 + ProxyPassReverse ^/(harvest.*)$ http://<master-ip>/$1 + ProxyPassMatch ^/(harvest.*)$ http://<master-ip>/$1 + ProxyPassReverse ^/(api/3/action/package.*)$ http://<master-ip>/$1 + ProxyPassMatch ^/(api/3/action/package.*)$ http://<master-ip>/$1 + ProxyPassMatch ^/(organization/edit/[^/]+)$ http://<master-ip>/$1 + ProxyPassReverse ^/(organization/edit/[^/]+)$ http://<master-ip>/$1 + ProxyPass /organization/new http://<interne-IP-Master>/organization/new +ProxyPassReverse /organization/new http://<interne-IP-Master>/organization/new +</VirtualHost \ No newline at end of file diff --git a/ckanext/odsh/tests/test_env.py b/ckanext/odsh/tests/test_env.py index c17ec642d599fc0f3e30f87a02dd25198b3ec4ee..8eb866e485d32c8a1ea57eefc74613c147859e41 100644 --- a/ckanext/odsh/tests/test_env.py +++ b/ckanext/odsh/tests/test_env.py @@ -8,6 +8,7 @@ import os import sys import ConfigParser from collections import OrderedDict +from urlparse import urlsplit expected_commit = '8cd9576884cae6abe50a27c891434cb9fe87ced2' @@ -178,3 +179,91 @@ class TestEnv: # version = readUrl(url+'/api/3/action/resource_qv4yAI2rgotamXGk98gJ').strip() # # version = checkConfig('ckanext.odsh.version') # assert version == expected_commit, "wrong version: {was}!={exp}".format(was=version, exp=expected_commit) + + def test_routes(self): + # if isMaster(): + # return + + expexted_rules = \ + """ ProxyPass /dataset/new http://10.61.47.219/dataset/new + ProxyPassReverse /dataset/new http://10.61.47.219/dataset/new + ProxyPassMatch ^/(dataset/delete/[^/]+)$ http://10.61.47.219/$1 + ProxyPassReverse ^/(dataset/delete/[^/]+)$ http://10.61.47.219/$1 + ProxyPassMatch ^/(dataset/edit/[^/]+)$ http://10.61.47.219/$1 + ProxyPassReverse ^/(dataset/edit/[^/]+)$ http://10.61.47.219/$1 + ProxyPassReverse /dataset http://141.91.184.90/dataset + ProxyPassReverse /dataset http://141.91.184.90/dataset + ProxyPass /dataset/new_resource http://10.61.47.219/dataset/new_resource + ProxyPassReverse /dataset/new_resource http://141.91.184.90/dataset/new_resource + ProxyPassReverse /dataset/new_resource http://141.91.184.90/dataset/new_resource + #ProxyPass /api/i18n/de http://141.91.184.90/api/i18n/de + ProxyPassReverse ^/uploads/group/(.*)$ http://10.61.47.219/uploads/group/$1 + ProxyPassMatch ^/uploads/group/(.*)$ http://10.61.47.219/uploads/group/$1 + ProxyPassReverse ^/(dataset/[^/]+/resource/[^/]+/download/[^/]+)$ http://141.91.184.90/$1 + ProxyPassMatch ^/(dataset/[^/]+/resource/[^/]+/download/[^/]+)$ http://141.91.184.90/$1 + ProxyPassReverse ^/(dataset/[^/]+/resource/[^/]+)$ http://10.61.47.219/$1 + ProxyPassMatch ^/(dataset/[^/]+/resource/[^/]+/)$ http://10.61.47.219/$1 + ProxyPassMatch ^/(dataset/[^/]+/resource_data/[^/]+)$ http://10.61.47.219/$1 + ProxyPassReverse ^/(dataset/[^/]+/resource_data/[^/]+)$ http://10.61.47.219/$1 + ProxyPassMatch ^/(dataset/[^/]+/resource/[^/]+/new_view[^/]*)$ http://10.61.47.219/$1 + ProxyPassReverse ^/(dataset/[^/]+/resource/[^/]+/new_view[^/]*)$ http://10.61.47.219/$1 + ProxyPassMatch ^/(harvest.*)$ http://141.91.184.90/$1 + ProxyPassReverse /harvest http://141.91.184.90/harvest + ProxyPass /harvest http://141.91.184.90/harvest + ProxyPassReverse ^/(harvest.*)$ http://141.91.184.90/$1 + ProxyPassReverse ^/(api/3/action/package.*)$ http://10.61.47.219/$1 + ProxyPassMatch ^/(api/3/action/package.*)$ http://10.61.47.219/$1 + ProxyPass /api/action/package_create http://10.61.47.219/api/action/package_create + ProxyPassReverse /api/action/package_create http://10.61.47.219/api/action/package_create + ProxyPass /api/action/resource_create http://10.61.47.219/api/action/resource_create + ProxyPassReverse /api/action/resource_create http://10.61.47.219/api/action/resource_create + ProxyPassMatch ^/(organization/edit/[^/]+)$ http://10.61.47.219/$1 + ProxyPassReverse ^/(organization/edit/[^/]+)$ http://10.61.47.219/$1 + ProxyPass /organization/new http://<interne-IP-Master>/organization/new + ProxyPassReverse /organization/new http://<interne-IP-Master>/organization/new + ProxyPassReverse /organization http://<interne-IP-Master>/organization + ProxyPassReverse ^/(organization/edit/[^/]+)$ http://<interne-IP-Master>/$1 + + # ProxyPass /datarequest http://10.61.47.219/datarequest + # ProxyPassReverse /datarequest http://10.61.47.219/datarequest + """ + + expected = self._parse_rules(expexted_rules.splitlines()) + + # with open('ckan_default.conf', 'r') as aconfig: + with open('/etc/apache2/sites-enabled/ckan_default.conf', 'r') as aconfig: + lines = aconfig.readlines() + # pdb.set_trace() + current = self._parse_rules(lines, check_host=True) + if len(expected.symmetric_difference(current)) > 0: + diff = expected.difference(current) + if len(diff) > 0: + print('WARNING: missing routes:') + for r in sorted(diff, key=lambda tup: tup[1]): + print('{cmd} {source} {target}'.format( + cmd=r[0], source=r[1], target='http://<interne-IP-Master>'+r[2])) + diff = current.difference(expected) + if len(diff) > 0: + print('WARNING: found unexpected routes:') + for r in sorted(diff, key=lambda tup: tup[1]): + print('{cmd} {source} {target}'.format( + cmd=r[0], source=r[1], target='<target>'+r[2])) + + def _parse_rules(self, lines, check_host=False): + rules = set(['ProxyPassMatch', 'ProxyPassReverse', 'ProxyPass']) + ret = [] + hosts = set() + for line in lines: + tokens = filter(lambda t: t.strip(), line.strip().split(' ')) + if not tokens or tokens[0] not in rules: + continue + assert len(tokens) == 3 + # for token in tokens: + # print(token) + f = urlsplit(tokens[2]) + ret.append((tokens[0], tokens[1], f.path)) + hosts.add(f.netloc) + if check_host and len(hosts) > 1: + print('WARNING: found multiple target hosts: {hosts}'.format( + hosts=', '.join(hosts))) + return set(ret)