Selenoid-UI with selenoid accessibility through nginx

Question

Selenoid-UI with selenoid accessibility through nginx

patjae opened this issue 5 years ago · 7 comments

Hello,

I'm trying to get complete setup of selenoid + selenoid-ui running behind a nginx reverse-proxy

having:
host1: running selenoid + selenoid-ui as docker containers
host2: nginx

On host1:

sudo mkdir -p /opt/aerokube-cm
sudo nano /opt/aerokube-cm/bash
sudo chmod 744 /opt/aerokube-cm/bash
cd /opt/aerokube-cm/
sudo ./bash

I got the cm script prepared and continue with this file

I initiated the selenoid initialization and start on host1:
sudo ./cm selenoid start --vnc --tmpfs 512 --args "-limit 10 -save-all-logs"

curl http://localhost:4444
You are using Selenoid 1.10.0!

I initiated the selenoid-ui initialization and start on host1:
sudo ./cm selenoid-ui start

curl http://localhost:8080
shows some valid response stating it is the Selenoid UI

At this point I configured nginx on host2, made the UI accessible, creating port-forwarding for certain endpoints of selenoid and the selenoid-ui accessible (just to check, if this is an issue caused by my docker run commands I run into with the next steps, but it seems to be not working entirely)

I cleaned up the containers, because I want to add additional configuration to the containers:

sudo docker stop selenoid
sudo docker rm selenoid
sudo docker stop selenoid-ui
sudo docker rm selenoid-ui

I prepared the following folders:

sudo mkdir -p /opt/aerokube-selenoid/etc/selenoid
sudo mkdir -p /opt/aerokube-selenoid/opt/selenoid/logs
sudo mkdir -p /opt/aerokube-selenoid/opt/selenoid/video

Reused the browsers.json copied from the default installation:
sudo nano /opt/aerokube-selenoid/etc/selenoid/browsers.json

Execute a docker run command for the backend/hub:

sudo docker run --detach \
    -p 4444:4444 \
    --name selenoid \
    --restart always \
    -e TZ=Etc/UTC \
    -e enableVNC=true \
    --volume /var/run/docker.sock:/var/run/docker.sock \
    --volume /opt/aerokube-selenoid/etc/selenoid:/etc/selenoid/:ro \
    --volume /opt/aerokube-selenoid/opt/selenoid/logs:/opt/selenoid/logs/ \
    --volume /opt/aerokube-selenoid/opt/selenoid/video:/opt/selenoid/video/ \
    -e OVERRIDE_VIDEO_OUTPUT_DIR=/opt/aerokube-selenoid/opt/selenoid/video \
    aerokube/selenoid:1.10.0 \
    -limit 10 \
    -save-all-logs \
    -conf /etc/selenoid/browsers.json

Check it:
curl http://localhost:4444
You are using Selenoid 1.10.0!

Execute a docker run command for the UI:

sudo docker run --detach \
    -p 11001:8080 \
    --name selenoid-ui \
    --restart always \
    --volume /var/run/docker.sock:/var/run/docker.sock \
    aerokube/selenoid-ui:1.10.0 \
    --selenoid-uri 'http://172.17.0.1:4444'

Check it:
curl http://localhost:8080
shows some valid response stating it is the Selenoid UI

My nginx provides a certificate terminating at this nginx instance, therefore urls are accessible via HTTPS/443.
I wanted to make the selenoid-ui accessible via the nginx on host2 on a subfolder/path, e.g. https://host2.internal.domain/selenoid-ui
but I read through several issues, and it seems that this is not working. So for now I try to make it accessible through the root "/" path, but I have to configure all additional paths manually, because there are already some other paths ("/tool1", "/tool2"...) defined in the nginx configuration.

nginx configuration on host2:

upstream tool-server {
    server host1:8443;
    keepalive 20;
}

upstream tool-server-selenoid {
    server host1:4444;
    keepalive 20;
}

upstream tool-server-selenoid-ui {
    server host1:11001;
    keepalive 20;
}


server {
    listen       80;
    listen  [::]:80;
    listen       443 ssl;
    listen  [::]:443 ssl;
    server_name  localhost;

    ssl_certificate /etc/nginx/ssl/host2.internal.domain.crt;
    ssl_certificate_key /etc/nginx/ssl/host2.internal.domain.key;

    client_max_body_size 2000M;

    # selenoid-ui
    location / {
       proxy_pass http://tool-server-selenoid-ui;
    }

    # selenoid backend - hub
    location /wd {
       proxy_pass http://tool-server-selenoid;
    }

    # selenoid backend - wss
    location /ws {
       proxy_pass http://tool-server-selenoid;
    }

    #location /status {
    #   proxy_pass http://tool-server-selenoid;
    #}

    location /video {
       proxy_pass http://tool-server-selenoid;
    }

    location /logs {
       proxy_pass http://tool-server-selenoid;
    }

    location /download {
       proxy_pass http://tool-server-selenoid;
    }

    location /clipboard {
       proxy_pass http://tool-server-selenoid;
    }

    location /devtools {
      
       proxy_pass http://tool-server-selenoid;
    }

    location /vnc {
      proxy_pass http://tool-server-selenoid;
    }

    location /ping {
       proxy_pass http://tool-server-selenoid;
    }
    
    location /events {
       proxy_pass http://tool-server-selenoid;
    }

    location /tool1 {
        proxy_pass https://tool-server/tool1;
        proxy_ssl_trusted_certificate /etc/nginx/ssl/host1.pem;
    }

   # location /selenoid-ui {
   #     proxy_pass http://tool-server-selenoid-ui;
   # }

    #error_page  404              /404.html;

    # redirect server error pages to the static page /50x.html
    #
    error_page   500 502 503 504  /50x.html;
    location = /50x.html {
        root   /usr/share/nginx/html;
    }
}

The selenoid-ui is accessible using https://host2.internal.domain/

Statuses are flickering between reloads:
status 1: SSE = ISSUE; SELENOID = UNKNOWN

status 2: SSE = UNKNOWN; SELENOID = CONNECTED

I think this is related to the proxy_pass for /status (I commented out, seems that both selenoid and selenoid-ui provide this endpoint, and if I activate it, e.g. the browser list in CAPABILITIES view does not load and it tells me # selenium: "http://selenoid-uri:4444" instead of # selenium: "http://172.17.0.1:4444")

If I press "Create Session", e.g. for chrome: 81.0, it shows the loading animation "..." with the ID, and another docker container was created on host 1.
But in the ui this process never finishes and I have to refresh manually. Than I find this newly created manual session in STATS.

If I access the session, e.g. https://host2.internal.domain/#/sessions/ it tells me
VNC DISCONNECTED
(on the left)

and
Initialize...

Connecting to wss://host2.internal.domain/ws/logs/...
Disconnected
(on the right)

Assuming that I really need those other targets in the nginx config, what am I doing wrong? do I have to proxy_pass additional URL paths or am I mixing things up of selenoid-ui and what goes to selenoid (hub/backend)?

Thank you

Edit:
I made some changes to nginx configuration, please see my additional comment.

Answer 1 · 2020-05-11T19:38:34.000Z

Gone through all my debugging iterations and found two issues that brought some progress:

Change the nginx config:
change this

location /events {
       proxy_pass http://tool-server-selenoid;
    }

to

location /events {
       proxy_pass http://tool-server-selenoid-ui;
    }

This seems to fix some status issues, status stays as
status 2: SSE = UNKNOWN; SELENOID = CONNECTED

While i was expecting this to be the selenoid and not the selenoid-ui path needed here, the logs looked good

And change
this
server_name localhost;
to
server_name host2.internal.domain;
and this

# selenoid backend - wss
    location /ws {
       proxy_pass http://tool-server-selenoid;
    }

to

# selenoid backend - wss
    location /ws {

       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
       proxy_set_header Host $host;
       proxy_pass http://tool-server-selenoid-ui;
       proxy_http_version 1.1;
       proxy_read_timeout 3600s;
       proxy_send_timeout 3600s;
       proxy_set_header Upgrade $http_upgrade;
       proxy_set_header Connection "upgrade";
    }

Wow, unfortunately this seems to fix the websocket communication and some additional issues with status model visible in the UI.

Maybe somebody might me point to the right direction to get this running under a context path like "/selenoid-ui" or check if i covered all paths needed currently?

Thank you.

Answer 2 · 2020-05-12T07:21:20.000Z

@patjae for relative path to work I think we need to fix this one: #49

Answer 3 · 2020-05-12T07:30:53.000Z

@patjae for relative path to work I think we need to fix this one: #49

@vania-pooh Yes, I found this issue, but did not understand if something was solved or not and what is working or what could be a workaround because it was closed and reopened again. Ok, I take this as a confirmation that we have to handle it in another way for now.
Thank you!

Answer 4 · 2021-06-01T09:05:03.000Z

Hey, I have solved this issue by adjusting nginx config like this:

Selenoid-ui is accessible on selenoid.example.om
Selenoid is accessible on selenoid.example.om/wd/hub/

add_header X-Frame-Options SAMEORIGIN;
add_header X-Content-Type-Options nosniff;
add_header X-XSS-Protection "1; mode=block";

upstream websocket-servers {
    server 127.0.01:8080;
}

server {
    listen 443;
    server_name {{ domain }};

    ssl on;
    ssl_certificate         /etc/letsencrypt/live/{{ domain }}/fullchain.pem;
    ssl_certificate_key     /etc/letsencrypt/live/{{ domain }}/privkey.pem;
    ssl_trusted_certificate /etc/letsencrypt/live/{{ domain }}/fullchain.pem;

    ssl_session_cache shared:SSL:50m;
    ssl_session_timeout 5m;
    ssl_stapling on;
    ssl_stapling_verify on;

    ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
    ssl_ciphers "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4";

    ssl_dhparam /etc/nginx/dhparams.pem;
    ssl_prefer_server_ciphers on;

    root /var/www/{{ domain }};
    index index.html index.htm;

    location / {
        proxy_pass http://127.0.0.1:8080;

        proxy_http_version 1.1;

        proxy_set_header Connection 'upgrade';
        chunked_transfer_encoding off;

        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header Host $host;

        proxy_buffering off;
        proxy_cache off;

        proxy_connect_timeout 600;
        proxy_send_timeout 300;
        proxy_read_timeout 300;
    }
}

EDIT: Added websocket configuration, so VNC and logs are working.
ISSUE: SSE state is UNKNOWN, so no new events are displayed (refresh needed)
EDIT2: Updated location config and SSE works now - see SSE Unknown over NGINX and TLS #442
for more info.
ISSUE2:: It seemed like working, however after fixing SSE logs and VNC stopped working. Progress will be shared in the issue #442 .
EDIT3: Fixed ISSUE2 by setting proxy_set_header Connection to upgrade.

Answer 5 · 2021-09-23T16:26:22.000Z

Maybe this'll help somebody. To make websockets working in k8s cluster with nginx ingress controller (https://docs.nginx.com/nginx-ingress-controller/) it's enough to add annotation to ingress resource: nginx.org/websocket-services: moon where moon is the service name of selenoid-ui.
When usual nginx ingress is used, http-snippet + location-snippet can be used.

Answer 6 · 2023-08-09T06:33:00.000Z

Hey, I have solved this issue by adjusting nginx config like this:

Selenoid-ui is accessible on selenoid.example.om Selenoid is accessible on selenoid.example.om/wd/hub/

add_header X-Frame-Options SAMEORIGIN;
add_header X-Content-Type-Options nosniff;
add_header X-XSS-Protection "1; mode=block";

upstream websocket-servers {
    server 127.0.01:8080;
}

server {
    listen 443;
    server_name {{ domain }};

    ssl on;
    ssl_certificate         /etc/letsencrypt/live/{{ domain }}/fullchain.pem;
    ssl_certificate_key     /etc/letsencrypt/live/{{ domain }}/privkey.pem;
    ssl_trusted_certificate /etc/letsencrypt/live/{{ domain }}/fullchain.pem;

    ssl_session_cache shared:SSL:50m;
    ssl_session_timeout 5m;
    ssl_stapling on;
    ssl_stapling_verify on;

    ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
    ssl_ciphers "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4";

    ssl_dhparam /etc/nginx/dhparams.pem;
    ssl_prefer_server_ciphers on;

    root /var/www/{{ domain }};
    index index.html index.htm;

    location / {
        proxy_pass http://127.0.0.1:8080;

        proxy_http_version 1.1;

        proxy_set_header Connection 'upgrade';
        chunked_transfer_encoding off;

        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header Host $host;

        proxy_buffering off;
        proxy_cache off;

        proxy_connect_timeout 600;
        proxy_send_timeout 300;
        proxy_read_timeout 300;
    }
}

EDIT: Added websocket configuration, so VNC and logs are working. ISSUE: SSE state is UNKNOWN, so no new events are displayed (refresh needed) EDIT2: Updated location config and SSE works now - see SSE Unknown over NGINX and TLS #442 for more info. ISSUE2:: It seemed like working, however after fixing SSE logs and VNC stopped working. Progress will be shared in the issue #442 . EDIT3: Fixed ISSUE2 by setting proxy_set_header Connection to upgrade.

Works fine but need to add location to selenoid
upstream selenoid {
server 127.0.0.1:4444;
keepalive 20;
}

location /devtools {
proxy_pass http://selenoid;
}
without it get 404 error

Answer 7 · 2024-01-15T01:50:38.000Z

This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.