Skip to content
Snippets Groups Projects
Commit 9b902598 authored by Radim Peša's avatar Radim Peša
Browse files

Init commit

parent 4700ce8a
No related branches found
No related tags found
No related merge requests found
Showing
with 543 additions and 0 deletions
.env 0 → 100644
#Datahub
HOST=sandbox2.edirex.ics.muni.cz
EMAIL_ADMIN_USER=dataportal-edirex@ics.muni.cz
ADMIN_PASSWORD=edirex
#PhpMyAdmin
PMA_HOSTS=db, cbioportal-database
PMA_USER=root
PMA_PASSWORD=rootpwd
# MySQL
MYSQL_ROOT_PASSWORD=rootpwd
#MYSQL_HOST=localhost
DBHOST=db
MYSQL_PORT=3306
MYSQL_DATABASE=datahub
MYSQL_USER=dhuser
MYSQL_PASSWORD=dhpwd
# Dataportal
NEO4J_AUTH=neo4j/neo5j
EXTENSION_SCRIPT=/neo4j-data/neo4j-init.sh
NEO4J_dbms.allow_upgrade=true
NEO4J_dbms_allow__format__migration=true
NEO4J_ALLOW_STORE_UPGRADE=true
AUTH_PASS=1
# Galaxy
IMPORT_DIR=./galaxy/import
REFERENCE_DIR=./galaxy/rfdir
\ No newline at end of file
FROM ubuntu:20.04
RUN apt-get update
RUN DEBIAN_FRONTEND="noninteractive" apt-get -y install tzdata
ENV TZ=Europe/Prague
RUN apt-get install -y python3 python3-pip curl docker.io
RUN mkdir /api
COPY . /api
WORKDIR /api
RUN pip3 install -r requirements.txt
EXPOSE 5001
CMD ["python3", "api.py"]
\ No newline at end of file
import logging
import os
from app import app
from app.tools import set_status_value
if __name__ == '__main__':
if not os.path.isfile(app.config['STATUS_FILE']):
set_status_value("down")
logging.basicConfig(level=logging.INFO, handlers=[logging.FileHandler("log/flask.log"), logging.StreamHandler()])
app.run(debug=True, host='0.0.0.0', port=5001)
from flask import Flask
from flask_restful import Api
from config import Config
app = Flask(__name__)
app.config.from_object(Config)
app.app_context().push()
api = Api(app)
from app import routes
File added
File added
File added
File added
File added
#!/bin/bash
whoami
set -x
HOME_DIR=/home/pdxuser
COMPOSE_DIR=$HOME_DIR/sandbox-docker
DOCUMENTS=$HOME_DIR/Documents
IMPORT_FOLDER=/tmp/import
# Make sure emty ~/Documents directory exists
mkdir -p $DOCUMENTS
rm -r $DOCUMENTS/pdx.graphdb/*
mkdir -p $DOCUMENTS/pdx.graphdb
# Extract seed db to ~/Documents/pdx.graphdb
tar -zxvf $HOME_DIR/data4reset/neo4j-data/graph.tgz --directory $DOCUMENTS/pdx.graphdb
# Launch loader
cd $HOME_DIR/dataportal-sourcecode
java -jar indexer/target/indexer-1.0.0.jar load --group=EurOPDX --data-dir=$IMPORT_FOLDER/europdx
# Tar created N4J DB files
cd $HOME_DIR/Documents/pdx.graphdb
tar -czvf ../graph.tgz *
#Move graph.tgz to $COMPOSE_DIR/neo4j-data
mv -f ../graph.tgz $COMPOSE_DIR/neo4j-data
#Restart Sandbox stack
cd $COMPOSE_DIR
docker-compose down
docker volume rm sandbox-docker_neodata
docker-compose up -d
# DS - init DataHub
sleep 30
#cd ~/datahub-docker/api/
#./init_datahub.sh
docker exec datahub bash -c "sed -i 's/localhost/db/g' config.py"
docker exec datahub ./init_datahub_from_container.sh
cd $HOME_DIR/sandbox-api/app
./load-custom-study.sh
#!/bin/bash
export ID=`curl -X POST -H "Content-Type: application/json" -k https://localhost:5000/api/v1/tmplists/NKIX | jq '.tmplistid'`
if [ $ID = "null" ];
then
echo "There are no pdxmodels for NKIX study"
else
echo $ID
rm -r _studies/*
# create files for cBioPortal
/home/pdxuser/data-providing/cbio_client.py
# move the files to cBioPortal folder
rm -r /home/pdxuser/cbioportal-docker-compose/study/*
cp -r _studies/* /home/pdxuser/cbioportal-docker-compose/study/
# run cBioPortal's importer
cd /home/pdxuser/cbioportal-docker-compose/
./import_studies.sh
fi
#!/bin/bash
set -x
whoami
HOME_DIR=/home/pdxuser
COMPOSE_DIR=$HOME_DIR/sandbox-docker
CBIO_DIR=$HOME_DIR/cbioportal-docker-compose
#TEST_STUDY=https://gitlab.ics.muni.cz/europdx/sandbox/data/-/raw/master/test-study.zip?inline=false
# Just info
docker ps
docker images
# stop cbio docker stack
cd $CBIO_DIR
docker-compose down
docker volume prune -f
# stop dataportal stack
cd $COMPOSE_DIR
docker-compose down
docker volume prune -f
# Download and unarchive test-study.zip - import-user-data
#wget $TEST_STUDY -O /tmp/test-study.zip
#rm -r $COMPOSE_DIR/import-user-data/*
#unzip /tmp/test-study.zip -d $COMPOSE_DIR/import-user-data
#rm /tmp/test-study.zip
# Populate import-data
# TODO change source
#cp $HOME_DIR/data4reset/import-data/*.json $COMPOSE_DIR/import-data
# Populate neo4j-data
#TODO change source
cp $HOME_DIR/data4reset/neo4j-data/graph.tgz $COMPOSE_DIR/neo4j-data
# Run indexer?
# Start the dataportal stack
cd $COMPOSE_DIR
docker-compose up -d
# Start the cbio stack
cd $CBIO_DIR
./reset_cbioportal.sh
# Wait untill stack is up
attempt_counter=0
max_attempts=5
until $(curl --insecure --output /dev/null --silent --head --fail https://localhost); do
if [ ${attempt_counter} -eq ${max_attempts} ];then
echo "Max attempts reached"
exit 1
fi
printf '.'
attempt_counter=$(($attempt_counter+1))
sleep 5
done
# DS - init DataHub
sleep 30
#cd ~/datahub-docker/api/
#./init_datahub.sh
docker exec datahub bash -c "sed -i 's/localhost/db/g' config.py"
docker exec datahub ./init_datahub_from_container.sh
#Change status to running
import os
import tarfile
import shutil
import logging
from flask import jsonify, request, abort, Response
from redis import Redis
import rq
from app import app
from app.tools import init_folder, set_status_value, get_status_value
from flask_httpauth import HTTPTokenAuth
from config import Config
auth = HTTPTokenAuth(scheme='Bearer')
tokens = {
"edirex": "Administrator"
}
@auth.verify_token
def verify_token(token):
if token in tokens:
return tokens[token]
#@app.before_first_request
#def before_first_request_func():
# init_folder(app.config['UPLOAD_FOLDER'])
# init_folder(app.config['IMPORT_FOLDER'])
@app.errorhandler(404)
def page_not_found(e):
return '<h1>Error 404</h1><b>Sandbox API</b><p>More to be found in <a href="' + app.config['USAGE_INFO_ADDRESS'] + '"> documentation</a>.</p>', 404
@app.route('/')
@auth.login_required
def index():
return "Hello, {}!".format(auth.current_user())
@app.route(app.config['PREFIX'] + '/status', methods=['GET'])
@auth.login_required
def get_status():
"""
:return: sandbox status
"""
return jsonify(status=get_status_value())
@app.route(app.config['PREFIX'] + '/log/reset', methods=['GET'])
@auth.login_required
def get_reset_log():
"""
:return: sandbox reset log
"""
log = "***** Reset stdout log *****\n"
if os.path.isfile(app.config['RESET_STDOUT']):
with open(app.config['RESET_STDOUT'], 'r') as stdout_file:
log += stdout_file.read()
else:
log +="Nothing in stdout.\n"
log += "\n***** Reset stderr log *****\n"
if os.path.isfile(app.config['RESET_STDERR']):
with open(app.config['RESET_STDERR'], 'r') as stderr_file:
log += stderr_file.read()
else:
log +="Nothing in stderr.\n"
return log
@app.route(app.config['PREFIX'] + '/log/import', methods=['GET'])
@auth.login_required
def get_import_log():
"""
:return: sandbox import log
"""
log = "***** Import stdout log *****\n"
if os.path.isfile(app.config['IMPORT_STDOUT']):
with open(app.config['IMPORT_STDOUT'], 'r') as stdout_file:
log += stdout_file.read()
else:
log += "Nothing in stdout.\n"
log += "\n***** Import stderr log *****\n"
if os.path.isfile(app.config['IMPORT_STDERR']):
with open(app.config['IMPORT_STDERR'], 'r') as stderr_file:
log += stderr_file.read()
else:
log += "Nothing in stderr.\n"
return log
@app.route(app.config['PREFIX'] + '/import', methods=['POST'])
@auth.login_required
def import_data():
"""
Starts import of data from import directory
:return: sandbox status
"""
set_status_value("importing")
queue = rq.Queue('sandbox-api', connection=Redis.from_url('redis://'))
job = queue.enqueue('app.tasks.import_data', job_timeout=500) # Probably it will be necessary to increase the timeout if imported bigger data
# tgz_file_full_path = os.path.join(app.config['UPLOAD_FOLDER'], app.config['UPLOAD_FILE'])
# if os.path.isfile(tgz_file_full_path):
# tar = tarfile.open(tgz_file_full_path)
# tar.extractall(path=app.config['IMPORT_FOLDER'])
# tar.close()
# logging.info("Tgz file extracted.")
# else:
# resp = jsonify(status="failed", message="Tgz file not available.")
# resp.status_code = 400
# return resp
resp = jsonify(status="ok", message="Import initialized.")
resp.status_code = 200
return resp
@app.route(app.config['PREFIX'] + '/reset', methods=['POST'])
@auth.login_required
def reset():
"""
Starts reset of sandbox environment
:return: sandbox status
"""
set_status_value("resetting")
queue = rq.Queue('sandbox-api', connection=Redis.from_url('redis://'))
job = queue.enqueue('app.tasks.reset_sandbox')
logging.info("Reset initialized.")
resp = jsonify(status="ok", message="Reset initialized.")
resp.status_code = 200
return resp
@app.route(app.config['PREFIX'] + '/upload', methods=['POST'])
@auth.login_required
def upload_data():
"""
Uploads file with data.
:return: sandbox status
"""
# check if the post request has the file part
if 'file' not in request.files:
logging.error("No file part in the request")
resp = jsonify({'message': 'No file part in the request'})
resp.status_code = 400
return resp
file = request.files['file']
if file.filename == '':
logging.error("No file selected for upload.")
resp = jsonify({'message': 'No file selected for upload.'})
resp.status_code = 400
return resp
filename = file.filename
try:
os.makedirs(app.config['UPLOAD_FOLDER'],mode=0o755)
except FileExistsError:
pass
file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
logging.info("File uploaded.")
resp = jsonify({'message': 'File successfully uploaded'})
resp.status_code = 201
return resp
import time
import logging
import os
import subprocess
import tarfile
from app import app
from config import Config
from app.tools import init_folder, set_status_value, get_status_value
#def example(seconds):
# print('Starting task')
# for i in range(seconds):
# print(i)
# time.sleep(1)
# print('Task completed')
def import_data():
logging.basicConfig(level=logging.DEBUG, handlers=[logging.FileHandler("log/tasks.log"), logging.StreamHandler()])
logging.info("Starting tgz file extraction.")
tgz_file_full_path = os.path.join(app.config['UPLOAD_FOLDER'], app.config['UPLOAD_FILE'])
if os.path.isfile(tgz_file_full_path):
tar = tarfile.open(tgz_file_full_path)
tar.extractall(path=app.config['IMPORT_FOLDER'])
tar.close()
logging.info("Tgz file extracted.")
else:
logging.info("Tgz file not available.")
# TODO launch shell script
stdout_file = open(app.config['IMPORT_STDOUT'], 'w')
stderr_file = open(app.config['IMPORT_STDERR'], 'w')
# TODO launch shell script
proc = subprocess.run("./app/import.sh", stdout=stdout_file, stderr=stderr_file)
logging.info('Return code: ' + str(proc.returncode))
if proc.returncode == 0:
set_status_value("running")
logging.info("Setting status to running.")
else:
set_status_value("failed_import")
logging.info("Setting status to failed.")
def reset_sandbox():
logging.basicConfig(level=logging.DEBUG, handlers=[logging.FileHandler("log/tasks.log"), logging.StreamHandler()])
logging.info("Starting sandbox reset process.")
stdout_file = open(app.config['RESET_STDOUT'], 'w')
stderr_file = open(app.config['RESET_STDERR'], 'w')
# TODO launch shell script
proc = subprocess.run("./app/reset.sh", stdout=stdout_file, stderr=stderr_file)
logging.info("Sandbox reset process finished.")
logging.info('Reset bash return code: ' + str(proc.returncode))
if proc.returncode == 0:
set_status_value("running")
logging.info("Setting status to running.")
else:
set_status_value("failed_reset")
logging.info("Setting status to failed.")
import os
import json
from app import app
from config import Config
def init_folder(folder):
if not (os.path.isdir(folder)):
os.mkdir(folder)
else:
for filename in os.listdir(folder):
file_path = os.path.join(folder, filename)
try:
if os.path.isfile(file_path):
os.unlink(file_path)
elif os.path.isdir(file_path):
shutil.rmtree(file_path)
except Exception as e:
print('Failed to delete %s. Reason: %s' % (file_path, e))
def set_status_value(status_value):
if status_value in app.config['STATUS_VALUES']:
status_dict = {"status": status_value}
with open(app.config['STATUS_FILE'], 'w') as sf:
json.dump(status_dict, sf)
else:
raise Exception(status_value + " is not a valid status value.")
def get_status_value():
with open(app.config['STATUS_FILE'], 'r') as sf:
status_dict = json.load(sf)
return status_dict['status']
import os
basedir = os.path.abspath(os.path.dirname(__file__))
class Config(object):
PREFIX = '/api'
USAGE_INFO_ADDRESS = 'https://gitlab.ics.muni.cz/europdx/sandbox/sandbox-api/-/blob/master/README.md'
UPLOAD_FOLDER = '/tmp/upload'
UPLOAD_FILE = 'europdx.tgz'
STATUS_FILE = 'status.json'
STATUS_VALUES = ['running', 'importing', 'resetting', 'down', 'failed', 'failed_import', 'failed_reset']
IMPORT_FOLDER = '/tmp/import'
IMPORT_STDOUT = 'log/import_stdout.log'
IMPORT_STDERR = 'log/import_stderr.log'
RESET_STDOUT = 'log/reset_stdout.log'
RESET_STDERR = 'log/reset_stderr.log'
# Ignore everything in this directory
*
# Except this file
!.gitignore
\ No newline at end of file
aniso8601==8.0.0
click==7.1.2
Flask==1.1.2
Flask-HTTPAuth==4.1.0
Flask-RESTful==0.3.8
itsdangerous==1.1.0
Jinja2==2.11.2
MarkupSafe==1.1.1
pytz==2020.1
redis==3.5.3
rq==1.4.3
six==1.15.0
Werkzeug==1.0.1
#!/bin/bash
source venv/bin/activate
python3 api.py
#!/bin/bash
source venv/bin/activate
rq worker sandbox-api
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment