diff --git a/DOCKER_README.md b/DOCKER_README.md new file mode 100644 index 0000000..9878e5b --- /dev/null +++ b/DOCKER_README.md @@ -0,0 +1,157 @@ +# Neutron/Quark docker development environment + +## General Info: +Your user needs root privileges - at this point mostly for starting rabbitmq. + +**IMPORTANT NOTE:** The container network will not work while RAX Cisco Anyconnect VPN is connected. Disconnect the VPN before doing anything. + +## TLDR; +Git clone the quark repo and cd into the repo directory. +Run: + +``` +$ sudo bash docker_up.sh +``` + +Test the API with curl like so: + +``` +$ curl http://localhost:9696/v2.0/networks +``` + +Make changes to quark code and re-deploy: + +``` +$ sudo bash docker_reload.sh +``` + +To cleanup: + +``` +$ sudo bash docker_down.sh` +``` + +## General API request flow: +User request → wafflehaus → neutron → quark → db + xen + + +## Files: +> quark/ + +> docker_up.sh + +> docker_down.sh + +> docker_reload.sh + +> quark/infrastructure/docker/quark/Dockerfile + + +## Logs: +To view logs, tail -f this file: +> quark/quark_container_logs/neutron-server.log + +## Docker: +Install docker for your OS! + +## To build: +Modify Dockerfile, typically in infrastructure/docker/quark/Dockerfile +Run: + +``` +$ docker build -t raxuser/neutron:quark -f infrastructure/docker/quark/Dockerfile . +``` + +This will create an image. + +``` +$ docker images ← will show the image ID +``` + + +Run (note: sudo needs to get root privileges): + +``` +$ sudo bash docker_up.sh +``` + +This will create these containers: +> Mysql ← takes 30 sec + +> ELK + +> RabbitMQ ← requires root + +> Redis + +> Redis Sentinel + +> Neutron/Quark ← takes several minutes + + +If you want to view what’s going on: + +``` +$ docker ps → find the neutron/quark container ID +``` +``` +$ docker attach ID +``` + +This will start scrolling what’s going. +If after a few minutes you are back at the bash prompt - something went wrong. +Scroll up to see what exactly. Also, may check the log file mentioned above. +If all went well, you should see port 9696 listening (netstat -an | grep 9696). You should be able to query the neutron API, for example: + +``` +$ curl http://localhost:9696/v2.0/networks +``` + +You could also: + +``` +$ tail -f quark/quark_container_logs/neutron-server.log +``` + +Note: you will have to restart the above tail command every time you re-deploy neutron. + +Go to http://localhost:8083, click “Discover” tab, click “time picker” and pick “Today”. This will show all the logs for today in a parsed format. + +## To Make Code Changes: +Make your changes to the Quark files in place where you checked them out on your workstation. There is no need to copy files to the containers or back. +After the changes were made, run: + +``` +$ sudo bash docker_reload.sh +``` + +This will destroy the old the quark container, start a new quark container, and install the new neutron and quark code. +This operation takes a couple minutes. +After the rebuild is done, all should work. + +## Cleanup: +Get the IDs of the containers: + +``` +$ docker ps -a +``` + +Remove the unnecessary containers: + +``` +$ docker rm container_ID +``` + +List images and get IDs: + +``` +$ docker images +``` + +Remove unnecessary images: + +``` +$ docker rmi image_ID +``` + +End of document. diff --git a/docker.neutron.conf b/docker.neutron.conf index e73577c..44e10fb 100644 --- a/docker.neutron.conf +++ b/docker.neutron.conf @@ -1,4 +1,5 @@ [DEFAULT] +debug = true lock_path = $state_path/lock core_plugin = quark.plugin.Plugin auth_strategy = noauth @@ -7,6 +8,7 @@ rpc_backend = rabbit log_dir: "/var/log/neutron" use_syslog: "True" syslog_log_facility: "LOG_LOCAL0" +api_workers = 1 [DATABASE] connection = mysql://root:@docker-mysql/neutron?charset=utf8 @@ -18,7 +20,7 @@ max_overflow = 500 [QUARK] public_net_id=00000000-0000-0000-0000-000000000000 -net_driver=quark.drivers.unmanaged.UnmanagedDriver +net_driver = 'BASE,UNMANAGED,NVP,IRONIC' default_net_strategy='{"00000000-0000-0000-0000-000000000000": {"bridge": "publicnet", "subnets": {"4": "00000000-0000-0000-0000-000000000000", "6": "11111111-1111-1111-1111-111111111111"}}, "11111111-1111-1111-1111-111111111111": {"bridge": "servicenet", "subnets": {"4": "22222222-2222-2222-2222-222222222222", "6": "333333333-3333-3333-3333-333333333333"}}}' default_network_type=UNMANAGED # IPAM recycling interval, in seconds diff --git a/docker_reload.sh b/docker_reload.sh index 54b8cef..f6f399d 100755 --- a/docker_reload.sh +++ b/docker_reload.sh @@ -7,12 +7,14 @@ echo "Neutron/Quark: Complete" echo "-----------------------" echo " " +# NOTE(alexm): there was a request not to remove these log files on reload rm -rf ./quark_container_logs/* rm -rf ./quark_container_venv/* echo "Neutron/Quark: Starting Container..." -docker run -d -v $(pwd):/opt/quark -v ~/neutron:/opt/configs -v $(pwd)/quark_container_logs:/var/log/neutron -v $(pwd)/quark_container_venv:/opt/venv/lib/python2.7/site-packages/quark -p 9696:9696 --link mysql:docker-mysql --link kibana:docker-kibana --link rabbitmq:docker-rabbitmq --link redis-sentinel:docker-redis-sentinel --name quark stajkowski/quark -# docker run --entrypoint /bin/bash -v $(pwd):/opt/quark -v ~/neutron:/opt/configs -v $(pwd)/quark_container_logs:/var/log/neutron -v $(pwd)/quark_container_venv:/opt/venv -p 9696:9696 --link mysql:docker-mysql --link kibana:docker-kibana --link rabbitmq:docker-rabbitmq --link redis-sentinel:docker-redis-sentinel --name quark stajkowski/quark +# NOTE(alexm): may need to add -v ~/neutron:/opt/configs when quark is caught up with upstream neutron +docker run -d -v $(pwd):/opt/quark -v $(pwd)/quark_container_logs:/var/log/neutron -v $(pwd)/quark_container_venv:/opt/venv/lib/python2.7/site-packages/quark -p 9696:9696 --link mysql:docker-mysql --link kibana:docker-kibana --link rabbitmq:docker-rabbitmq --link redis-sentinel:docker-redis-sentinel --name quark raxuser/neutron:quark + echo "Neutron/Quark: Waiting for Neutron to Start..." # Need to wait for DB to standup sleep 5 diff --git a/docker_up.sh b/docker_up.sh index aa78424..9873d26 100755 --- a/docker_up.sh +++ b/docker_up.sh @@ -41,8 +41,8 @@ echo "------------------------------------------" echo " " echo "Neutron/Quark: Starting Container..." -docker run -d -v $(pwd):/opt/quark -v ~/neutron:/opt/configs -v $(pwd)/quark_container_logs:/var/log/neutron -v $(pwd)/quark_container_venv:/opt/venv/lib/python2.7/site-packages/quark -p 9696:9696 --link mysql:docker-mysql --link kibana:docker-kibana --link rabbitmq:docker-rabbitmq --link redis-sentinel:docker-redis-sentinel --name quark stajkowski/quark -# docker run --entrypoint /bin/bash -v $(pwd):/opt/quark -v ~/neutron:/opt/configs -v $(pwd)/quark_container_logs:/var/log/neutron -v $(pwd)/quark_container_venv:/opt/venv -p 9696:9696 --link mysql:docker-mysql --link kibana:docker-kibana --link rabbitmq:docker-rabbitmq --link redis-sentinel:docker-redis-sentinel --name quark stajkowski/quark +# NOTE(alexm): add -v ~/neutron:/opt/configs after quark is caught up with upstream neutron +docker run -t -d -v $(pwd):/opt/quark -v $(pwd)/quark_container_logs:/var/log/neutron -v $(pwd)/quark_container_venv:/opt/venv/lib/python2.7/site-packages/quark -p 9696:9696 --link mysql:docker-mysql --link kibana:docker-kibana --link rabbitmq:docker-rabbitmq --link redis-sentinel:docker-redis-sentinel --name quark raxuser/neutron:quark echo "Neutron/Quark: Waiting for Neutron to Start..." # Need to wait for DB to standup sleep 5 @@ -72,22 +72,22 @@ curl -X POST -H "Content-Type: application/json" -d '{"network": {"id": "0000000 echo " " echo "Creating Subnets..." echo " " -curl -X POST -H "Content-Type: application/json" -d '{"subnet": {"network_id": "00000000-0000-0000-0000-000000000000", "segment_id": "blah", "cidr": "10.1.0.0/16", "tenant_id": "derp", "ip_version": "4"}}' http://localhost:9696/v2.0/subnets +curl -X POST -H "Content-Type: application/json" -d '{"subnet": {"network_id": "00000000-0000-0000-0000-000000000000", "segment_id": "rax_segment_id", "cidr": "10.1.0.0/16", "tenant_id": "rax_tenant", "ip_version": "4"}}' http://localhost:9696/v2.0/subnets echo " " -curl -X POST -H "Content-Type: application/json" -d '{"subnet": {"network_id": "00000000-0000-0000-0000-000000000000", "segment_id": "blah", "cidr": "10.2.0.0/16", "tenant_id": "derp", "ip_version": "4"}}' http://localhost:9696/v2.0/subnets +curl -X POST -H "Content-Type: application/json" -d '{"subnet": {"network_id": "00000000-0000-0000-0000-000000000000", "segment_id": "rax_segment_id", "cidr": "10.2.0.0/16", "tenant_id": "rax_tenant", "ip_version": "4"}}' http://localhost:9696/v2.0/subnets echo " " -curl -X POST -H "Content-Type: application/json" -d '{"subnet": {"network_id": "00000000-0000-0000-0000-000000000000", "segment_id": "blah", "cidr": "10.3.0.0/16", "tenant_id": "derp", "ip_version": "4"}}' http://localhost:9696/v2.0/subnets +curl -X POST -H "Content-Type: application/json" -d '{"subnet": {"network_id": "00000000-0000-0000-0000-000000000000", "segment_id": "rax_segment_id", "cidr": "10.3.0.0/16", "tenant_id": "rax_tenant", "ip_version": "4"}}' http://localhost:9696/v2.0/subnets echo " " -curl -X POST -H "Content-Type: application/json" -d '{"subnet": {"network_id": "00000000-0000-0000-0000-000000000000", "segment_id": "blah", "cidr": "10.4.0.0/16", "tenant_id": "derp", "ip_version": "4"}}' http://localhost:9696/v2.0/subnets +curl -X POST -H "Content-Type: application/json" -d '{"subnet": {"network_id": "00000000-0000-0000-0000-000000000000", "segment_id": "rax_segment_id", "cidr": "10.4.0.0/16", "tenant_id": "rax_tenant", "ip_version": "4"}}' http://localhost:9696/v2.0/subnets echo " " echo "Create Networks: Complete - http://localhost:9696" echo "------------------------------------------" echo " MAC Range: AA:BB:CC" echo " NETWORK: 00000000-0000-0000-0000-000000000000" -echo " CIDR: 10.1.0.0/16 SEGMENT_ID: blah TENANT_ID: derp" -echo " CIDR: 10.2.0.0/16 SEGMENT_ID: blah TENANT_ID: derp" -echo " CIDR: 10.3.0.0/16 SEGMENT_ID: blah TENANT_ID: derp" -echo " CIDR: 10.4.0.0/16 SEGMENT_ID: blah TENANT_ID: derp" +echo " CIDR: 10.1.0.0/16 SEGMENT_ID: rax_segment_id TENANT_ID: rax_tenant" +echo " CIDR: 10.2.0.0/16 SEGMENT_ID: rax_segment_id TENANT_ID: rax_tenant" +echo " CIDR: 10.3.0.0/16 SEGMENT_ID: rax_segment_id TENANT_ID: rax_tenant" +echo " CIDR: 10.4.0.0/16 SEGMENT_ID: rax_segment_id TENANT_ID: rax_tenant" echo "------------------------------------------" echo " COMPLETE!!" echo "------------------------------------------" diff --git a/infrastructure/docker/quark/Dockerfile b/infrastructure/docker/quark/Dockerfile index d1e7622..89c27ad 100644 --- a/infrastructure/docker/quark/Dockerfile +++ b/infrastructure/docker/quark/Dockerfile @@ -17,14 +17,20 @@ RUN mkdir /opt/configs /opt/neutron /opt/quark /opt/venv /opt/wafflehaus /opt/wa RUN cd /opt/venv && virtualenv . --distribute # Clone Neutron & Wafflehaus -RUN git clone https://github.com/openstack/neutron /opt/neutron +# NOTE(alexm): enable this after quark is up to date with upstream neutron +# RUN git clone https://github.com/openstack/neutron /opt/neutron RUN git clone https://github.com/roaet/wafflehaus.git /opt/wafflehaus RUN git clone https://github.com/roaet/wafflehaus.neutron /opt/wafflehaus.neutron +# NOTE(alexm): enable this when quark is caught up to upstream neutron # Pull Neutron and Install -RUN echo "cd /opt/neutron && git pull" >> ~/entrypoint.sh -RUN echo "source /opt/venv/bin/activate && pip install -U -r /opt/neutron/requirements.txt" >> ~/entrypoint.sh -RUN echo "cd /opt/neutron && source /opt/venv/bin/activate && python setup.py develop" >> ~/entrypoint.sh +# RUN echo "cd /opt/neutron && git pull" >> ~/entrypoint.sh +# RUN echo "source /opt/venv/bin/activate && pip install -U -r /opt/neutron/requirements.txt" >> ~/entrypoint.sh +# RUN echo "cd /opt/neutron && source /opt/venv/bin/activate && python setup.py develop" >> ~/entrypoint.sh + +# NOTE(alexm): remove this when quark is caught up to upstream neutron +# Install Neutron +RUN echo "source /opt/venv/bin/activate && pip install -U http://tarballs.openstack.org/neutron/neutron-10.0.1.tar.gz#egg=neutron" >> ~/entrypoint.sh # Install Quark RUN echo "source /opt/venv/bin/activate && pip install -U -r /opt/quark/requirements.txt" >> ~/entrypoint.sh @@ -47,7 +53,8 @@ RUN echo "source /opt/venv/bin/activate && pip install -U ipdb" >> ~/entrypoint. RUN echo "cp /opt/quark/docker.neutron.conf ~/neutron.conf" >> ~/entrypoint.sh RUN echo "cp /opt/quark/docker.apipaste.ini ~/api-paste.ini" >> ~/entrypoint.sh RUN echo "cp /opt/quark/docker.policy.json ~/policy.json" >> ~/entrypoint.sh -RUN echo "cp /opt/configs/* ~/" >> ~/entrypoint.sh +# NOTE(alexm): enable this after quark is up to date with upstream neutron +# RUN echo "cp /opt/configs/* ~/" >> ~/entrypoint.sh RUN echo "source /opt/venv/bin/activate && quark-db-manage --config-file ~/neutron.conf upgrade head" >> ~/entrypoint.sh RUN echo "source /opt/venv/bin/activate && neutron-server --config-file ~/neutron.conf" >> ~/entrypoint.sh @@ -56,4 +63,4 @@ RUN chmod +x ~/entrypoint.sh EXPOSE 9696 -ENTRYPOINT ["/bin/bash", "-c", "~/entrypoint.sh"] \ No newline at end of file +ENTRYPOINT ["/bin/bash", "-c", "~/entrypoint.sh"] diff --git a/quark/db/migration/alembic/versions/79b768afed65_rename_tenant_id_indexes.py b/quark/db/migration/alembic/versions/79b768afed65_rename_tenant_id_indexes.py index 3c39ab2..cbdef64 100644 --- a/quark/db/migration/alembic/versions/79b768afed65_rename_tenant_id_indexes.py +++ b/quark/db/migration/alembic/versions/79b768afed65_rename_tenant_id_indexes.py @@ -13,7 +13,7 @@ down_revision = '271cce54e15b' from alembic import op import sqlalchemy as sa -from neutron.api.v2 import attributes as attr +from neutron_lib.db import constants as db_const _INSPECTOR = None @@ -116,7 +116,7 @@ def alter_column(table, column): old_name = 'tenant_id' new_name = 'project_id' - coltype = sa.String(attr.TENANT_ID_MAX_LEN) + coltype = sa.String(db_const.PROJECT_ID_FIELD_SIZE) op.alter_column( table_name=table, diff --git a/test-requirements.txt b/test-requirements.txt index 0044422..f2775f0 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -13,5 +13,8 @@ discover==0.4.0 unittest2==1.0.1 nose==1.3.7 +# NOTE(alexm): pinning the neutron version until quark is +# up to date with the upstream neutron code # Pulls latest neutron, but it should be pinned in packaged deploys -http://tarballs.openstack.org/neutron/neutron-master.tar.gz#egg=neutron +# http://tarballs.openstack.org/neutron/neutron-master.tar.gz#egg=neutron +http://tarballs.openstack.org/neutron/neutron-10.0.0.tar.gz#egg=neutron