...
Code Block |
---|
$ sudo apt-get install python-setuptools git libpq-dev postgresql python-dev postfix nginx-full uwsgi uwsgi-plugin-python libffi-dev
$ sudo easy_install pip
$ sudo pip install virtualenv
$ sudo adduser --disabled-password --home /var/opt/norduni ni
|
...
Download neo4j-community from http://neo4j.com/download/. NORDUnet and SUNET run 2.1.8. 2.3.2 has been tested and did not work as expected.
Code Block |
---|
$ tar xvfz neo4j-community-2.1.78-unix.tar.gz $ sudo mv neo4j-community-2.1.78 /var/opt/. $ sudo ln -s /var/opt/neo4j-community-2.1.78 /var/opt/neo4j-community $ cd /var/opt/neo4j-community $ sudo ./bin/neo4j-installer install |
...
Code Block |
---|
$ . norduni_environment/bin/activate
$ pip install -r norduni/requirements/prod.txt
|
Django settings
Change the django settings.
Code Block |
---|
$ cd norduni/src/niweb/niweb/ $ cp generic_settings.pydotenv settings.pyenv $ vi settings.pyenv |
Change at least the database settingsThe following settings need to be changed.
Code Block |
---|
REPORTS_TO=
DB_PASSWORD=
DEFAULT_FROM_EMAIL=
EMAIL_HOST=
SECRET_KEY= |
Check if your settings are ok.
Code Block |
---|
$ python manage.py syncdb
$ python manage.py migrate apps.noclook
$ python manage.py migrate actstream
$ python manage.py migrate tastypie
$ python manage.py collectstatic
$ python manage.py runserver
|
Now you should be able connect to the machine with your browser on http://localhost:8000 and see the NOCLook app index page.
Deploying NOCLook
uwsgi
Create a uwsgi configuration file.
Code Block |
---|
$ sudo vi /etc/uwsgi/apps-available/noclook.ini The following configuration should be a good start. [uwsgi] # Django-related settings plugins = python protocol = uwsgi # the base directory (full path) chdir# Database settings DATABASES = { 'default': { 'ENGINE': 'django.db.backends.postgresql_psycopg2', 'NAME': 'norduni', 'USER': 'ni', 'PASSWORD': 'secret', 'HOST': 'localhost' } } |
Check if your settings are ok.
Code Block |
---|
$ python manage.py syncdb
$ python manage.py migrate apps.noclook
$ python manage.py migrate actstream
$ python manage.py migrate tastypie
$ python manage.py runserver
|
Now you should be able connect to the machine with your browser on http://localhost:8000 and see the NOCLook app index page.
Deploying NOCLook
uwsgi
Create a uwsgi configuration file.
Code Block |
---|
$ sudo vi /etc/uwsgi/apps-available/noclook.ini The following configuration should be a good start. [uwsgi] # Django-related settings plugins = python protocol = uwsgi # the base directory (full path) chdir = /var/opt/norduni/norduni/src/niweb/ # Django's wsgi file wsgi-file = /var/opt/norduni/norduni/src/niweb/niweb/wsgi.py env = DJANGO_SETTINGS_MODULE=niweb.settings.prod # the virtualenv (full path) home = /var/opt/norduni/norduni_environment # logging daemonize = /var/optlog/norduniuwsgi/norduni/src/niweb/app/noclook.log # Django's wsgi file wsgi-fileprocess-related settings # master master = /var/opt/norduni/norduni/src/niweb/niweb/wsgi.py envtrue # maximum number of worker processes processes = 5 #threads = DJANGO_SETTINGS_MODULE=niweb.settings 2 max-requests = 5000 # the virtualenvsocket (use the full path) home to be safe socket = /var/opt/norduni/norduni_environment127.0.0.1:8001 # logging daemonizeclear environment = /var/log/uwsgi/app/noclook.log # process-related settings # master masteron exit vacuum = true # maximum number of worker processes processes = 5 #threads = 2 max-requests = 5000 # the socket (use the full path to be safe socket = |
Link the configuration in to the correct directory.
Code Block |
---|
sudo ln -s /etc/uwsgi/apps-available/noclook.ini /etc/uwsgi/apps-enabled/noclook.ini |
Make temp dir and log dir writable by the uwsgi user (www-data on ubuntu)
Code Block |
---|
sudo chown -R ni:www-data /tmp/django_cache
sudo chmod -R g+w /tmp/django_cache
sudo chown -R ni:www-data /var/opt/norduni/norduni/src/niweb/logs/
sudo chmod -R g+w /var/opt/norduni/norduni/src/niweb/logs/ |
nginx
Setup new dhparam file 2048 should suffice, but if you like you can go with 4096 instead:
Code Block |
---|
$ sudo openssl dhparam -out /etc/ssl/dhparams.pem 2048 |
Configure nginx.
Code Block | ||
---|---|---|
| ||
$ sudo vi /etc/nginx/sites-available/default The following configuration should be a good start. upstream django { server 127.0.0.1:8001 ; # clearfor a environmentweb onport exit vacuum socket } server { listen = true |
Link the configuration in to the correct directory.
Code Block |
---|
sudo ln -s /etc/uwsgi/apps-available/noclook.ini /etc/uwsgi/apps-enabled/noclook.ini |
nginx
Configure nginx.
Code Block | ||
---|---|---|
| ||
$ sudo vi /etc/nginx/sites-available/default The following configuration should be a good start. upstream django { server 127.0.0.1:8001; # for a web port socket 80; listen [::]:80; server_name ni.nordu.net; rewrite ^ https://$server_name$request_uri? permanent; } server { listen listen443; 80; listen listen [::]:443 default ipv6only=on; ## listen for ipv6 ssl [::]:80on; ssl_certificate /etc/ssl/ni_nordu_net.crt; serverssl_certificate_name ni.nordu.net;key /etc/ssl/ni_nordu_net.key; rewrite # https://cipherli.st ssl_prefer_server_ciphers on; ^ https://$server_name$request_uri? permanent; } server { listen 443 ssl_protocols TLSv1 TLSv1.1 TLSv1.2; ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH"; listen [::]:443 default ipv6only=on; ## listen for ipv6ssl_session_cache shared:SSL:10m; ssl_ecdh_curve secp384r1; ssl on; ssl_certificate _dhparam /etc/ssl/ni_nordu_net.crt;dhparams.pem; sslserver_certificate_key /etc/ssl/ni_nordu_net.keyname ni.nordu.net; # PFS settings from http://people.adams.edu/~cdmiller/posts/NSA_SSL_settings_for_nginx_and_apache/location /static/ { # NOTE: these settings alias exclude Win XP with IE 6 /var/opt/norduni/norduni/src/niweb/niweb/static/; # autoindex on; ssl_protocols TLSv1 TLSv1.1 TLSv1.2; access_log off; expires ssl_prefer_server_ciphers on; 30d; ssl_ciphers DHE-RSA-CAMELLIA256-SHA:DHE-RSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA:DES-CBC3-SHA:!ADH:!aNULL;} location / { include server_name ni.nordu.net; /etc/nginx/uwsgi_params; location /static/ { proxy_set_header Host root /var/opt/norduni/norduni/src/niweb/sitestatic/ $host; autoindexproxy_set_header on; X-Real-IP access_log off$remote_addr; expiresproxy_set_header X-Forwarded-For 30d; } $proxy_add_x_forwarded_for; location / { proxy_set_header X-Forwarded-Proto include /etc/nginx/uwsgi_params $scheme; proxy_set_header proxy_redirect Host off; uwsgi_pass $hostdjango; } proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_redirect off; uwsgi_pass django; } } |
SAML SP
If you want to set up NOCLook as a SAML SP you need to install the following packages and Python modules.
Code Block |
---|
$ sudo apt-get install libffi-dev xmlsec1
$ sudo -u ni -i
$ . norduni_environment/bin/activate
$ pip install djangosaml2 |
}
|
SAML SP
If you want to set up NOCLook as a SAML SP you need to install the following packages and Python modules.
Code Block |
---|
$ sudo apt-get install libffi-dev xmlsec1
$ sudo -u ni -i
$ . norduni_environment/bin/activate
$ pip install djangosaml2 |
You then need to uncomment the lines in settings.py that imports and sets up djangosaml2. You also have to create a pysaml2 configuration.
All this is best described in the documentation at https://pypi.python.org/pypi/djangosaml2.
Local saml metadata
To speed up login you can use local metadata. This metadata still needs to be updated and verified, and for that you can use https://github.com/NORDUnet/metadata-updater
You need to configure djangosaml2 to use local metadata, and you will have to add the meta-dataupdater to cron, preferably by running crontab -e as the ni user. Once an hour is reasonable, once a day can be ok, once a week might be tiresome when the cert expiresYou then need to uncomment the lines in settings.py that imports and sets up djangosaml2. You also have to create a pysaml2 configuration.
All this is best described in the documentation at https://pypi.python.org/pypi/djangosaml2.
Collecting and processing network data
...
Code Block | ||
---|---|---|
| ||
{
"host": {
"csv_producer": {
"address": "",
"area": "",
"city": "",
"comment": "",
"country": "",
"floor": "",
"latitude": "",
"longitude": "",
"meta_type": "",
"name": "",
"node_type": "",
"owner_id": "",
"postcode": "",
"responsible_for": "",
"room": "",
"site_type": "",
"telenor_subscription_id": ""
},
"name": "",
"version": 1
}
}
|
The consumer script should only be run once as it does not update the sites, only creates new.
The JSON file directory is then inserted in to the database using noclook_site_csv_consumer.py.
Change the path at the top of the script to be able to import norduni_client.py.
Then run:
Code Block |
---|
python noclook_site_csv_consumer.py -D /path/to/site_files/json
|
Daily database update
1
}
}
|
The consumer script should only be run once as it does not update the sites, only creates new.
The JSON file directory is then inserted in to the database using noclook_site_csv_consumer.pyThe producers are run with a cron job and the script noclook_consumer.py is used to run the three inserting/updating scripts (noclook_juniper_consumer.py, noclook_alcatel_consumer.py and noclook_nmap_consumer.py).
Change the path at the top of the script to be able to import norduni_client.py.
...
to be able to import norduni_client.py.
Then run:
Code Block |
---|
python noclook_site_csv_consumer.py -C template.conf -I |
Setting up a local/development NOCLook
D /path/to/site_files/json
|
Daily database update
The producers are run with a cron job and the script noclook_consumer.py is used to run the three inserting/updating scripts (noclook_juniper_consumer.py, noclook_alcatel_consumer.py and noclook_nmap_consumer.py).
Change the path at the top of the script to be able to import norduni_client.py.
Code Block |
---|
[data]
juniper_conf = /path/to/juniper/json
nmap_services = /path/to/nmap/json
alcatel_isis = /path/to/alcate/json
noclook = #Used for loading backup.
|
Then run:
Code Block |
---|
python noclook_consumer.py -C template.conf -I
|
Setting up a local/development NOCLook
Code Block |
---|
# Clone a convenience repo
$ git clone https://github.com/NORDUnet/norduni-developer
$ cd norduni-developer
# Start dependencies
$ ./start.sh
# Clone NOCLook project repo
$ git clone https://git.nordu.net/norduni.git
$ cd norduni |
Code Block |
$ git clone https://git.nordu.net/norduni.git # Download neo4j docker image and start it $ docker pull tpires/neo4j $ docker run -d -v /path_to_repo/norduni/docker/neo4j.properties:/var/lib/neo4j/conf/neo4j.properties -v /opt/docker/neo4jdata:/var/lib/neo4j/data -p 7474:7474 tpires/neo4j # Create the indexes with curl $ curl -D - -H "Content-Type: application/json" --data '{"name" : "node_auto_index","config" : {"type" : "fulltext","provider" : "lucene"}}' -X POST http://localhost:7474/db/data/index/node/ $ curl -D - -H "Content-Type: application/json" --data '{"name" : "relationship_auto_index","config" : {"type" : "fulltext","provider" : "lucene"}}' -X POST http://localhost:7474/db/data/index/relationship/ # Create a virtualenv and activate it $ virtualenv env $ . env/bin/activate # Install the python packages $ pip install paver $ pip install -r /path_to_repo/requirementsrequirements/dev.txt # Create a settings.py from the template /path_to/repo/src/niweb/niweb file $ cp /path_to_repo/src/niweb/niweb/generic_settings.py /path_to/repo/src/niweb/dotenv src/niweb/settings.pydevenv # Sync the db $ python /path_to_repo/src/niweb/manage.py syncdb $ python /path_to_repo/src/niweb/manage.py migrate apps.noclook # Run the app $ python /path_to_repo/src/niweb/manage.py migrate actstream $ python /path_to_repo/src/niweb/manage.py migrate tastypie runserver |
Upgrading to newest versions
This is the general procedure for upgrading to newest version of norduni.
Code Block |
---|
# stash current local changes and update $ git stash $ git pull origin master $ git stash apply # Run the appmigrations $ python /path_to_repo/src/niweb/manage.py runservermigrate # OptionalPip postgres instead of sqlite3, don't forget to change database settings in settings.py. # Download postgres docker image and start it $ docker pull orchardup/postgresql $ docker run -d -p 5432:5432 -e POSTGRESQL_USER=ni -e POSTGRESQL_PASS=docker -e POSTGRESQL_DB=norduni -v /opt/docker/postgresql_data/:/var/lib/postgresql/ orchardup/postgresql update requirements $ pip install -U -r requirements/prod.txt # Collect statics $ python /path_to_repo/src/niweb/manage.py collectstatic # Restart uwsgi $ sudo services uwsgi restart |