Trying to run a miminally modified Dockerfile
Closed this issue · 8 comments
seanodea commented
Hi,
I have a docker file that I got from Dockerfile.sample. It builds but hypercorn outputs this error. When I try with gunicorn, I get the same issue. Did I miss a doc or something?
[2020-06-08 08:58:10 +0000] [10] [INFO] Starting gunicorn 20.0.4
[2020-06-08 08:58:10 +0000] [10] [INFO] Listening at: http://0.0.0.0:5000 (10)
[2020-06-08 08:58:10 +0000] [10] [INFO] Using worker: sync
[2020-06-08 08:58:10 +0000] [16] [INFO] Booting worker with pid: 16
[2020-06-08 08:58:11 +0000] [16] [ERROR] Exception in worker process
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/gunicorn/arbiter.py", line 583, in spawn_worker
worker.init_process()
File "/usr/local/lib/python3.8/site-packages/gunicorn/workers/base.py", line 119, in init_process
self.load_wsgi()
File "/usr/local/lib/python3.8/site-packages/gunicorn/workers/base.py", line 144, in load_wsgi
self.wsgi = self.app.wsgi()
File "/usr/local/lib/python3.8/site-packages/gunicorn/app/base.py", line 67, in wsgi
self.callable = self.load()
File "/usr/local/lib/python3.8/site-packages/gunicorn/app/wsgiapp.py", line 49, in load
return self.load_wsgiapp()
File "/usr/local/lib/python3.8/site-packages/gunicorn/app/wsgiapp.py", line 39, in load_wsgiapp
return util.import_app(self.app_uri)
File "/usr/local/lib/python3.8/site-packages/gunicorn/util.py", line 358, in import_app
mod = importlib.import_module(module)
File "/usr/local/lib/python3.8/importlib/__init__.py", line 127, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "<frozen importlib._bootstrap>", line 1014, in _gcd_import
File "<frozen importlib._bootstrap>", line 991, in _find_and_load
File "<frozen importlib._bootstrap>", line 975, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 671, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 783, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "/home/avwx/avwx_api/__init__.py", line 3, in <module>
from avwx_api import api, views, status
File "/home/avwx/avwx_api/api/__init__.py", line 1, in <module>
from . import current, gfs, station
File "/home/avwx/avwx_api/api/current.py", line 14, in <module>
class MetarFetch(Report):
File "/usr/local/lib/python3.8/site-packages/quart_openapi/pint.py", line 286, in decorator
self._add_resource(func_or_viewcls, path, methods, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/quart_openapi/pint.py", line 199, in _add_resource
super().add_url_rule(path, endpoint, view_func, methods, # pylint: disable=no-member
TypeError: add_url_rule() got multiple values for argument 'provide_automatic_options'
I tried directly on my machine, installing the requirements with pip -Ruv requirements.txt. I followed the docker file basically and ended up with the same error:
seanodea@aurvandr ~/code/flsb/flsb-flask/avwx-api (master)$ hypercorn avwx_api:app --bind 0.0.0.0:5000 -c python:hypercorn_config.py
Process Process-2:
Process Process-1:
Process Process-3:
Traceback (most recent call last):
Traceback (most recent call last):
File "/usr/lib/python3.8/multiprocessing/process.py", line 315, in _bootstrap
self.run()
File "/usr/lib/python3.8/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/home/seanodea/.local/lib/python3.8/site-packages/hypercorn/asyncio/run.py", line 163, in asyncio_worker
app = load_application(config.application_path)
File "/home/seanodea/.local/lib/python3.8/site-packages/hypercorn/utils.py", line 92, in load_application
module = import_module(import_name)
File "/usr/lib/python3.8/importlib/__init__.py", line 127, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "<frozen importlib._bootstrap>", line 1014, in _gcd_import
File "<frozen importlib._bootstrap>", line 991, in _find_and_load
File "<frozen importlib._bootstrap>", line 975, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 671, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 783, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "/home/seanodea/code/fl-statusboard/flsb-flask/avwx-api/avwx_api/__init__.py", line 3, in <module>
from avwx_api import api, views
File "/home/seanodea/code/fl-statusboard/flsb-flask/avwx-api/avwx_api/api/__init__.py", line 1, in <module>
from . import current, gfs, station
File "/home/seanodea/code/fl-statusboard/flsb-flask/avwx-api/avwx_api/api/current.py", line 14, in <module>
class MetarFetch(Report):
File "/home/seanodea/.local/lib/python3.8/site-packages/quart_openapi/pint.py", line 286, in decorator
self._add_resource(func_or_viewcls, path, methods, *args, **kwargs)
File "/home/seanodea/.local/lib/python3.8/site-packages/quart_openapi/pint.py", line 199, in _add_resource
super().add_url_rule(path, endpoint, view_func, methods, # pylint: disable=no-member
TypeError: add_url_rule() got multiple values for argument 'provide_automatic_options'
File "/usr/lib/python3.8/multiprocessing/process.py", line 315, in _bootstrap
self.run()
File "/usr/lib/python3.8/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/home/seanodea/.local/lib/python3.8/site-packages/hypercorn/asyncio/run.py", line 163, in asyncio_worker
app = load_application(config.application_path)
File "/home/seanodea/.local/lib/python3.8/site-packages/hypercorn/utils.py", line 92, in load_application
module = import_module(import_name)
File "/usr/lib/python3.8/importlib/__init__.py", line 127, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "<frozen importlib._bootstrap>", line 1014, in _gcd_import
File "<frozen importlib._bootstrap>", line 991, in _find_and_load
File "<frozen importlib._bootstrap>", line 975, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 671, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 783, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "/home/seanodea/code/fl-statusboard/flsb-flask/avwx-api/avwx_api/__init__.py", line 3, in <module>
from avwx_api import api, views
File "/home/seanodea/code/fl-statusboard/flsb-flask/avwx-api/avwx_api/api/__init__.py", line 1, in <module>
from . import current, gfs, station
File "/home/seanodea/code/fl-statusboard/flsb-flask/avwx-api/avwx_api/api/current.py", line 14, in <module>
class MetarFetch(Report):
File "/home/seanodea/.local/lib/python3.8/site-packages/quart_openapi/pint.py", line 286, in decorator
self._add_resource(func_or_viewcls, path, methods, *args, **kwargs)
File "/home/seanodea/.local/lib/python3.8/site-packages/quart_openapi/pint.py", line 199, in _add_resource
super().add_url_rule(path, endpoint, view_func, methods, # pylint: disable=no-member
TypeError: add_url_rule() got multiple values for argument 'provide_automatic_options'
Traceback (most recent call last):
File "/usr/lib/python3.8/multiprocessing/process.py", line 315, in _bootstrap
self.run()
File "/usr/lib/python3.8/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/home/seanodea/.local/lib/python3.8/site-packages/hypercorn/asyncio/run.py", line 163, in asyncio_worker
app = load_application(config.application_path)
File "/home/seanodea/.local/lib/python3.8/site-packages/hypercorn/utils.py", line 92, in load_application
module = import_module(import_name)
File "/usr/lib/python3.8/importlib/__init__.py", line 127, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "<frozen importlib._bootstrap>", line 1014, in _gcd_import
File "<frozen importlib._bootstrap>", line 991, in _find_and_load
File "<frozen importlib._bootstrap>", line 975, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 671, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 783, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "/home/seanodea/code/fl-statusboard/flsb-flask/avwx-api/avwx_api/__init__.py", line 3, in <module>
from avwx_api import api, views
File "/home/seanodea/code/fl-statusboard/flsb-flask/avwx-api/avwx_api/api/__init__.py", line 1, in <module>
from . import current, gfs, station/home/seanodea
File "/home/seanodea/code/fl-statusboard/flsb-flask/avwx-api/avwx_api/api/current.py", line 14, in <module>
class MetarFetch(Report):
File "/home/seanodea/.local/lib/python3.8/site-packages/quart_openapi/pint.py", line 286, in decorator
self._add_resource(func_or_viewcls, path, methods, *args, **kwargs)
File "/home/seanodea/.local/lib/python3.8/site-packages/quart_openapi/pint.py", line 199, in _add_resource
super().add_url_rule(path, endpoint, view_func, methods, # pylint: disable=no-member
TypeError: add_url_rule() got multiple values for argument 'provide_automatic_options'
Process Process-4:
Traceback (most recent call last):
File "/usr/lib/python3.8/multiprocessing/process.py", line 315, in _bootstrap
self.run()
File "/usr/lib/python3.8/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/home/seanodea/.local/lib/python3.8/site-packages/hypercorn/asyncio/run.py", line 163, in asyncio_worker
app = load_application(config.application_path)
File "/home/seanodea/.local/lib/python3.8/site-packages/hypercorn/utils.py", line 92, in load_application
module = import_module(import_name)
File "/usr/lib/python3.8/importlib/__init__.py", line 127, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "<frozen importlib._bootstrap>", line 1014, in _gcd_import
File "<frozen importlib._bootstrap>", line 991, in _find_and_load
File "<frozen importlib._bootstrap>", line 975, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 671, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 783, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "/home/seanodea/code/fl-statusboard/flsb-flask/avwx-api/avwx_api/__init__.py", line 3, in <module>
from avwx_api import api, views
File "/home/seanodea/code/fl-statusboard/flsb-flask/avwx-api/avwx_api/api/__init__.py", line 1, in <module>
from . import current, gfs, station
File "/home/seanodea/code/fl-statusboard/flsb-flask/avwx-api/avwx_api/api/current.py", line 14, in <module>
class MetarFetch(Report):
File "/home/seanodea/.local/lib/python3.8/site-packages/quart_openapi/pint.py", line 286, in decorator
self._add_resource(func_or_viewcls, path, methods, *args, **kwargs)
File "/home/seanodea/.local/lib/python3.8/site-packages/quart_openapi/pint.py", line 199, in _add_resource
super().add_url_rule(path, endpoint, view_func, methods, # pylint: disable=no-member
TypeError: add_url_rule() got multiple values for argument 'provide_automatic_options'
devdupont commented
Could you send me a pip freeze? I believe this may be an issue with a quart/pint library update.
seanodea commented
I'm using your repo as a git submodule to mine which contains my own Dockerfile, and a few patches(currently unapplied) to create a /status URL that puts out Mongo: true, Pgsql: false depending on the status. This image is going to run in heroku so it takes the PORT env which is always random and rerouted by the host on container up.
Dockerfile:
# Start from the official Python 3.8 container
FROM python:3.8.3-buster
RUN apt-get update
RUN apt-get install netcat-openbsd -y
# Set the main working directory
WORKDIR /home/avwx
# Set the service credentials as environment variables
ENV MONGO_URI=$MONGODB_URI \
LOG_KEY=$ROLLBAR_KEY \
PSQL_URI=$DATABASE_URL
# Install the require Python packages
COPY avwx-api/requirements.txt /home/avwx/requirements.txt
RUN pip install -U pip
RUN pip install -Ur avwx-api/requirements.txt --no-cache-dir --compile
# Copy the application code
COPY avwx-api/avwx_api /home/avwx/avwx_api
COPY status.py /home/avwx/avwx_api/
COPY status.patch /home/avwx/avwx_api/
WORKDIR /home/avwx/avwx_api
#RUN cat status.patch | patch
WORKDIR /home/avwx
# Run the application
ENTRYPOINT ["hypercorn"]
CMD ["avwx_api:app", "--bind", "0.0.0.0:${PORT}"]
On the container:
aiofiles==0.5.0
attrs==19.3.0
avwx-api-core @ git+git://github.com/avwx-rest/avwx-api-core.git@779cffd4826e43d12c0bf1e0fb43dca5d43c361a
avwx-engine==1.4.4
blinker==1.4
certifi==2020.4.5.2
chardet==3.0.4
click==7.1.2
dicttoxml==1.7.4
dnspython==1.16.0
geographiclib==1.50
geopy==1.22.0
gunicorn==20.0.4
h11==0.9.0
h2==3.2.0
hpack==3.0.0
hstspreload==2020.6.9
httpcore==0.9.1
httpx==0.13.3
Hypercorn==0.10.0
hyperframe==5.2.0
idna==2.9
itsdangerous==1.1.0
Jinja2==2.11.2
jsonschema==3.2.0
MarkupSafe==1.1.1
motor==2.1.0
numpy==1.18.5
priority==1.3.0
pymongo==3.10.1
pyrsistent==0.16.0
python-dateutil==2.8.1
PyYAML==5.3.1
Quart==0.12.0
quart-openapi==1.5.3
requests==2.23.0
rfc3986==1.4.0
rollbar==0.15.0
scipy==1.4.1
six==1.15.0
sniffio==1.1.0
toml==0.10.1
typing-extensions==3.7.4.2
urllib3==1.25.9
voluptuous==0.11.7
Werkzeug==1.0.1
wsproto==0.15.0
xmltodict==0.12.0
On my workstation:
aiofiles==0.5.0
attrs==19.3.0
Automat==0.8.0
avwx-api-core==0.1.0
avwx-engine==1.4.4
blinker==1.4
cached-property==1.5.1
certifi==2019.11.28
chardet==3.0.4
Click==7.0
cloud-init==20.1
colorama==0.4.3
command-not-found==0.3
configobj==5.0.6
constantly==15.1.0
cryptography==2.8
dbus-python==1.2.16
dicttoxml==1.7.4
distro==1.4.0
distro-info===0.23ubuntu1
dnspython==1.16.0
docker==4.1.0
docker-compose==1.25.0
dockerpty==0.4.1
docopt==0.6.2
entrypoints==0.3
Flask==1.1.1
geographiclib==1.50
geopy==1.22.0
gunicorn==20.0.4
h11==0.9.0
h2==3.2.0
hpack==3.0.0
hstspreload==2020.6.5
httpcore==0.9.1
httplib2==0.14.0
httpx==0.13.3
Hypercorn==0.10.0
hyperframe==5.2.0
hyperlink==19.0.0
idna==2.8
importlib-metadata==1.5.0
incremental==16.10.1
itsdangerous==1.1.0
Jinja2==2.10.1
jsonpatch==1.22
jsonpointer==2.0
jsonschema==3.2.0
keyring==18.0.1
language-selector==0.1
launchpadlib==1.10.13
lazr.restfulclient==0.14.2
lazr.uri==1.0.3
MarkupSafe==1.1.0
more-itertools==4.2.0
motor==2.1.0
netifaces==0.10.4
numpy==1.18.5
oauthlib==3.1.0
pathspec==0.7.0
priority==1.3.0
pyasn1==0.4.2
pyasn1-modules==0.2.1
PyGObject==3.36.0
PyHamcrest==1.9.0
pyinotify==0.9.6
PyJWT==1.7.1
pymacaroons==0.13.0
pymongo==3.10.1
PyNaCl==1.3.0
pyOpenSSL==19.0.0
pyrsistent==0.15.5
pyserial==3.4
python-apt==2.0.0
python-dateutil==2.8.1
python-debian===0.1.36ubuntu1
PyYAML==5.3.1
Quart==0.12.0
quart-openapi==1.5.3
requests==2.22.0
requests-unixsocket==0.2.0
rfc3986==1.4.0
rollbar==0.15.0
scipy==1.4.1
SecretStorage==2.3.1
service-identity==18.1.0
simplejson==3.16.0
six==1.14.0
sniffio==1.1.0
ssh-import-id==5.10
systemd-python==234
texttable==1.6.2
toml==0.10.1
Twisted==18.9.0
typing-extensions==3.7.4.2
ubuntu-advantage-tools==20.3
ufw==0.36
unattended-upgrades==0.1
urllib3==1.25.8
voluptuous==0.11.7
wadllib==1.3.3
websocket-client==0.53.0
Werkzeug==1.0.1
wsproto==0.15.0
xmltodict==0.12.0
yamllint==1.20.0
zipp==1.0.0
zope.interface==4.7.1
seanodea commented
Here is the modification I intend to make once it runs successfully:
status.patch
diff -ruN __init__.py __init__.py
--- __init__.py 2020-06-08 03:11:34.198730200 -0500
+++ __init__.py 2020-06-08 03:08:33.188631300 -0500
@@ -1,3 +1,3 @@
from avwx_api.app_config import app
-from avwx_api import api, views
+from avwx_api import api, views, status
status.py
"""
App Status
"""
from pymongo import MongoClient
import time
import psycopg2
#import avwx_api.handle.gfs as handle
#from avwx_api import app
#from avwx_api.api.base import Report, Parse
@app.route('/hw')
def hello():
return "Hello World!"
@app.route("/status")
def Status():
print("{mongo_connection: " + str(mongo_test()) + ", postgres_connection: " + str(postgres_test()) + "}")
def mongo_test():
try:
mongo_uri = os.env.get('MONGO_URI', "localhost:1111")
client = MongoClient(host = ["localhost:1111"], serverSelectionTimeoutMS = 2000)
client.server_info() # will throw an exception
return True
except:
return False
def postgres_test():
try:
postgres_uri = os.env.get('PSQL_URI', "localhost:1111")
conn = psycopg2.connect("dbname='mydb' user='myuser' host='my_ip' password='mypassword' connect_timeout=1 ")
conn.close()
return True
except:
return False
seanodea commented
Here is the portion of my cicd file that is failing to start the application.
flsb-test:
variables:
PORT: 5000
CHECKSTRING: "Hello World!"
CHECKURL: "http://localhost:$PORT/hw"
extends: .localtest
script:
- cd /home/avwx
- pip freeze || pip3 freeze
- gunicorn(hypercorn too) avwx_api:app --bind 0.0.0.0:$PORT &
- export starttime=`date +%s`
- while ! /bin/nc.openbsd -v 127.0.0.1 $PORT -w 1 && [ $(($(date +%s) - 120)) -lt $starttime ]; do sleep 5;echo "waiting on ports to be up...";done
- if [ $(wget -q $CHECKURL -O - | grep "${CHECKSTRING}" | wc -l) -eq 0 ];then echo "Failed to find ${CHECKSTRING} in output";exit 1;fi
devdupont commented
This turns out to be an issue where quart changed some parameters but quart-openapi hasn't been updated to work with it yet. I've submitted an issue/fix and updated the quart version number in the meantime. Fetching the update should fix your issue.
Looking at your status.py code, I'd highly recommend using async connection libraries from MongoDB and Postgres instead of pymongo and psycopg2 since you're running in an async context. An instance of MotorClient is already available at app.mdb for you to use. You can use the asyncpg library for Postgres.
seanodea commented
Thank you for your help and your work on this!!!
seanodea commented
Here is my new status.py:
"""
App Status
"""
import time, os, asyncio, asyncpg
import motor.motor_asyncio
from avwx_api import app #for MorotClient, a mongo db async module
@app.route('/hw')
async def hello():
return "Hello World!"
@app.route("/status")
async def Status():
if os.environ.get('MONGO_URI') != "":
url = os.environ.get('MONGO_URI')
else:
url = os.environ.get('MONGODB_URI')
try:
client = motor.motor_asyncio.AsyncIOMotorClient(url)
db = client.get_default_database()
mongostat = True
except:
mongostat = False
if os.environ.get('PSQL_URI') != "":
url = os.environ.get('PSQL_URI')
else:
url = os.environ.get('DATABASE_URL')
query = "SELECT * FROM pg_settings WHERE name = 'port';"
try:
conn = await asyncpg.connect(url)
if callable(query):
await query(conn)
else:
await conn.execute(query)
await conn.close()
pgstat = True
except:
pgstat = False
return "{status: 'up', mongo_connection: " + str(mongostat) + ", postgres_connection: " + str(pgstat) + ", port: " + str(os.environ.get('PORT')) + "}"
devdupont commented
You may find this to be a bit faster. This aio.gather allows multiple async calls to be made at once, so you don't have to wait for the mongo check to complete before testing the sql connection. Moved some other things to global since they won't change during runtime.
import os
import asyncio as aio
import asyncpg
from avwx_api import app
QUERY = "SELECT * FROM pg_settings WHERE name = 'port';"
DB_URL = os.environ.get('PSQL_URI') or os.environ.get('DATABASE_URL')
PORT = os.environ.get('PORT')
async def mongo_is_up() -> bool:
"""
Returns True if there is an active connection to MongoDB
"""
try:
await app.mdb.server_info()
return True
except:
return False
async def psql_is_up() -> bool:
"""
Returns True if there is an active connection to Postgres
"""
try:
conn = await asyncpg.connect(DB_URL)
await conn.execute(QUERY)
await conn.close()
return True
except:
return False
@app.route('/hw')
def hello():
return "Hello World!"
@app.route("/status")
async def status():
"""
Returns server status information
"""
results = await aio.gather(mongo_is_up(), psql_is_up())
return {
"status": "up",
"mongo": results[0],
"postgres": results[1],
"port": PORT,
}