Varnish not supporting in magento 1.9.2.4
Kailash0593 opened this issue · 3 comments
Kailash0593 commented
Hello,
I have spent almost a week trying to configure my Magento website with varnish turpentine but unable to make it work :(
Below is the complete details of my magneto varnish nginx and turpentine configuration
Magento v : 1.9.2.4
php v : 7.2
ubuntu : 16.04 Xenial
varnish : 4.1.1
turpentine : 0.7.5
nginx : 1.15.8
/etc/default/varnish
DAEMON_OPTS=" -a :80 \
-T localhost:6082 \
-f /etc/varnish/default.vcl \
-S /etc/varnish/secret \
-p feature=+esi_ignore_other_elements \
-p cli_buffer=16384 \
-p vcc_allow_inline_c=on \
-s malloc,256m"
/etc/systemd/system/varnish.service
[Unit]
Description=Varnish Cache, a high-performance HTTP accelerator
[Service]
Type=forking
LimitNOFILE=131072
LimitMEMLOCK=85983232
LimitCORE=infinity
ExecStart=/usr/sbin/varnishd -p vcc_allow_inline_c=on -p feature=+esi_disable_xml_check,+esi_ignore_other_elements -p cli_buffer=16384 -a :80 -T localhost:6082 -f /etc/varnish/default.vcl -S /etc/varnish/secret -s malloc,256m
ExecReload=/usr/share/varnish/reload-vcl
[Install]
WantedBy=multi-user.target
/etc/varnish/default.vcl
backend default {
.host = "127.0.0.1";
.port = "8080";
}
sub vcl_recv {
}
sub vcl_backend_response {
}
sub vcl_deliver {
}
Copy Pasting my full nginx conf code just to be sure that any other config is causing a problem or not?
server {
listen 80;
#listen [::]:80 ipv6only=on default_server;
root /var/www/html;
index index.php index.html index.htm;
server_name subdomain.example.com;
###########Google Page Speed###########
#######################################
#pagespeed on;
# Needs to exist and be writable by nginx. Use tmpfs for best performance.
#pagespeed FileCachePath /var/ngx_pagespeed_cache;
# Ensure requests for pagespeed optimized resources go to the pagespeed handler
# and no extraneous headers get set.
#location ~ "\.pagespeed\.([a-z]\.)?[a-z]{2}\.[^.]{10}\.[^.]+" {
#add_header "" "";
#}
#location ~ "^/pagespeed_static/" { }
#location ~ "^/ngx_pagespeed_beacon$" { }
#######################################
#######################################
location ^~ /dev/ { return 403; }
location ~ \.php$ {
try_files $uri =404;
fastcgi_split_path_info ^(.+\.php)(/.+)$;
fastcgi_pass unix:/var/run/php/php7.2-fpm.sock;
fastcgi_index index.php;
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
include fastcgi_params;
}
##################################################
### GZIP compression ###
gzip_disable "msie6";
gzip on;
gzip_comp_level 5;
gzip_min_length 256;
gzip_proxied any;
gzip_vary on;
gzip_buffers 16 8k;
gzip_http_version 1.1;
gzip_types
application/atom+xml
application/javascript
application/json
application/ld+json
application/manifest+json
application/rss+xml
application/vnd.geo+json
application/vnd.ms-fontobject
application/x-font-ttf
application/x-web-app-manifest+json
application/xhtml+xml
application/xml
font/opentype
image/bmp
image/svg+xml
image/x-icon
text/cache-manifest
text/css
text/plain
text/vcard
text/vnd.rim.location.xloc
text/vtt
text/x-component
text/x-cross-domain-policy;
####################################################
##########Leverage Browser Caching##################
location ~* \.(png|jpg|jpeg|ico|gif|svg|pdf|eot|otf|ttf|woff|woff2)$ {
expires 365d;
add_header Pragma public;
add_header Cache-Control "public";
}
location ~* \.(css|js)\$ {
expires 7d;
access_log off;
add_header Cache-Control "public";
}
####################################################
####################Cookie free headers#############
#disabled bcz it dosent allow to set cookie for admin
#fastcgi_hide_header Set-Cookie;
####################################################
location /api {
rewrite ^/api/rest /api.php?type=rest last;
rewrite ^/api/v2_soap /api.php?type=v2_soap last;
rewrite ^/api/soap /api.php?type=soap last;
}
}
I have also installed google page speed 1.13.35.1-beta but its not in use as i have commented the code in my nginx file as copy pasted above
MAGENTO_ROOT/var/default.vcl
vcl 4.0;
C {#
include < stdlib.h > #include < stdio.h > #include < time.h > #include < pthread.h >
static pthread_mutex_t lrand_mutex = PTHREAD_MUTEX_INITIALIZER;
void generate_uuid(char * buf) {
pthread_mutex_lock( & lrand_mutex);
long a = lrand48();
long b = lrand48();
long c = lrand48();
long d = lrand48();
pthread_mutex_unlock( & lrand_mutex);
sprintf(buf, "frontend=%08lx%04lx%04lx%04lx%04lx%08lx",
a,
b & 0xffff,
(b & ((long) 0x0fff0000) >> 16) | 0x4000,
(c & 0x0fff) | 0x8000,
(c & (long) 0xffff0000) >> 16,
d
);
return;
}
}
C
import std;
import directors;
backend
default {
.host = "127.0.0.1";
.port = "8080";
.first_byte_timeout = 300 s;
.between_bytes_timeout = 300 s;
}
backend admin {
.host = "127.0.0.1";
.port = "8080";
.first_byte_timeout = 21600 s;
.between_bytes_timeout = 21600 s;
}
acl crawler_acl {
"127.0.0.1";
}
acl debug_acl {
"111.119.252.175";
}
/* -- REMOVED
sub generate_session {
if (req.url ~ ".*[&?]SID=([^&]+).*") {
set req.http.X-Varnish-Faked-Session = regsub(
req.url, ".*[&?]SID=([^&]+).*", "frontend=\1");
} else {
C{
char uuid_buf [50];
generate_uuid(uuid_buf);
static const struct gethdr_s VGC_HDR_REQ_VARNISH_FAKED_SESSION =
{ HDR_REQ, "\030X-Varnish-Faked-Session:"};
VRT_SetHdr(ctx,
&VGC_HDR_REQ_VARNISH_FAKED_SESSION,
uuid_buf,
vrt_magic_string_end
);
}C
}
if (req.http.Cookie) {
std.collect(req.http.Cookie);
set req.http.Cookie = req.http.X-Varnish-Faked-Session +
"; " + req.http.Cookie;
} else {
set req.http.Cookie = req.http.X-Varnish-Faked-Session;
}
}
sub generate_session_expires {
C{
time_t now = time(NULL);
struct tm now_tm = *gmtime(&now);
now_tm.tm_sec += 886000;
mktime(&now_tm);
char date_buf [50];
strftime(date_buf, sizeof(date_buf)-1, "%a, %d-%b-%Y %H:%M:%S %Z", &now_tm);
static const struct gethdr_s VGC_HDR_RESP_COOKIE_EXPIRES =
{ HDR_RESP, "\031X-Varnish-Cookie-Expires:"};
VRT_SetHdr(ctx,
&VGC_HDR_RESP_COOKIE_EXPIRES,
date_buf,
vrt_magic_string_end
);
}C
}
-- */
sub vcl_init {}
sub vcl_recv {
if (req.url~"^(//media/|//skin/|/media/|/skin/|//js/|/js/|/)(?:(?:index|litespeed)\.php/)?officialnaari") {
set req.backend_hint = admin;
}
else {}
if (!true || req.http.Authorization ||
req.method!~"^(GET|HEAD|OPTIONS)$" ||
req.http.Cookie~"varnish_bypass=1") {
return (pipe);
}
if (false) {
set req.http.X - Varnish - Origin - Url = req.url;
}
set req.url = regsuball(req.url, "(.*)//+(.*)", "\1/\2");
if (req.http.Accept - Encoding) {
if (req.http.Accept - Encoding~"\*|gzip") {
set req.http.Accept - Encoding = "gzip";
}
else if (req.http.Accept - Encoding~"deflate") {
set req.http.Accept - Encoding = "deflate";
}
else {
unset req.http.Accept - Encoding;
}
}
if (req.url~"^(//media/|//skin/|/media/|/skin/|//js/|/js/|/)(?:(?:index|litespeed)\.php/)?") {
set req.http.X - Turpentine - Secret - Handshake = "1";
if (req.url~"^(//media/|//skin/|/media/|/skin/|//js/|/js/|/)(?:(?:index|litespeed)\.php/)?officialnaari") {
return (pipe);
}
if (req.http.Cookie~"\bcurrency=") {
set req.http.X - Varnish - Currency = regsub(
req.http.Cookie, ".*\bcurrency=([^;]*).*", "\1");
}
if (req.http.Cookie~"\bstore=") {
set req.http.X - Varnish - Store = regsub(
req.http.Cookie, ".*\bstore=([^;]*).*", "\1");
}
if (req.url~"/turpentine/esi/get(?:Block|FormKey)/") {
set req.http.X - Varnish - Esi - Method = regsub(
req.url, ".*/method/(\w+)/.*", "\1");
set req.http.X - Varnish - Esi - Access = regsub(
req.url, ".*/access/(\w+)/.*", "\1");
if (req.http.X - Varnish - Esi - Method == "esi" && req.esi_level == 0 &&
!(false || client.ip~debug_acl)) {
return (synth(403, "External ESI requests are not allowed"));
}
}
if (req.http.Cookie!~"frontend=" && !req.http.X - Varnish - Esi - Method) {
if (client.ip~crawler_acl ||
req.http.User - Agent~"^(?:ApacheBench/.*|.*Googlebot.*|.*APIs-Google.*|.*Mediapartners-Google.*|.*AdsBot-Google.*|JoeDog/.*|.*Siege/.*|magespeedtest\.com|Nexcessnet_Turpentine/.*|.*PTST.*|.*bingbot.*|.*msnbot.*|.*Baiduspider.*)$") {
set req.http.Cookie = "frontend=crawler-session";
}
else {
return (pipe);
}
}
if (true &&
req.url~".*\.(?:css|js|jpe?g|png|gif|ico|swf)(?=\?|&|$)") {
unset req.http.Cookie;
unset req.http.X - Varnish - Faked - Session;
set req.http.X - Varnish - Static = 1;
return (hash);
}
if (req.url~"^(//media/|//skin/|/media/|/skin/|//js/|/js/|/)(?:(?:index|litespeed)\.php/)?(?:officialnaari|api|cron\.php)" ||
req.url~"\?.*__from_store=") {
return (pipe);
}
if (true &&
req.url~"(?:[?&](?:__SID|XDEBUG_PROFILE)(?=[&=]|$))") {
return (pass);
}
if (req.url~"[?&](utm_source|utm_medium|utm_campaign|gclid|cx|ie|cof|siteurl)=") {
set req.url = regsuball(req.url, "(?:(\?)?|&)(?:utm_source|utm_medium|utm_campaign|gclid|cx|ie|cof|siteurl)=[^&]+", "\1");
set req.url = regsuball(req.url, "(?:(\?)&|\?$)", "\1");
}
if (true && req.url~"[?&](utm_source|utm_medium|utm_campaign|utm_content|utm_term|gclid|cx|ie|cof|siteurl)=") {
set req.url = regsuball(req.url, "(?:(\?)?|&)(?:utm_source|utm_medium|utm_campaign|utm_content|utm_term|gclid|cx|ie|cof|siteurl)=[^&]+", "\1");
set req.url = regsuball(req.url, "(?:(\?)&|\?$)", "\1");
}
if (false) {
set req.http.X - Varnish - Cache - Url = req.url;
set req.url = req.http.X - Varnish - Origin - Url;
unset req.http.X - Varnish - Origin - Url;
}
return (hash);
}
}
sub vcl_pipe {
unset bereq.http.X - Turpentine - Secret - Handshake;
set bereq.http.Connection = "close";
}
sub vcl_hash {
std.log("vcl_hash start");
if (true && req.http.X - Varnish - Static) {
std.log("hash_data static file - req.url: " + req.url);
hash_data(req.url);
if (req.http.Accept - Encoding) {
std.log("hash_data static file - Accept-Encoding: " + req.http.Accept - Encoding);
hash_data(req.http.Accept - Encoding);
}
std.log("vcl_hash end return lookup");
return (lookup);
}
if (false && req.http.X - Varnish - Cache - Url) {
hash_data(req.http.X - Varnish - Cache - Url);
std.log("hash_data - X-Varnish-Cache-Url: " + req.http.X - Varnish - Cache - Url);
} else {
hash_data(req.url);
std.log("hash_data - req.url: " + req.url);
}
if (req.http.Host) {
hash_data(req.http.Host);
std.log("hash_data - req.http.Host: " + req.http.Host);
} else {
hash_data(server.ip);
}
std.log("hash_data - req.http.Ssl-Offloaded: " + req.http.Ssl - Offloaded);
hash_data(req.http.Ssl - Offloaded);
if (req.http.X - Normalized - User - Agent) {
hash_data(req.http.X - Normalized - User - Agent);
std.log("hash_data - req.http.X-Normalized-User-Agent: " + req.http.X - Normalized - User - Agent);
}
if (req.http.Accept - Encoding) {
hash_data(req.http.Accept - Encoding);
std.log("hash_data - req.http.Accept-Encoding: " + req.http.Accept - Encoding);
}
if (req.http.X - Varnish - Store || req.http.X - Varnish - Currency) {
hash_data("s=" + req.http.X - Varnish - Store + "&c=" + req.http.X - Varnish - Currency);
std.log("hash_data - Store and Currency: " + "s=" + req.http.X - Varnish - Store + "&c=" + req.http.X - Varnish - Currency);
}
if (req.http.X - Varnish - Esi - Access == "private" &&
req.http.Cookie~"frontend=") {
std.log("hash_data - frontend cookie: " + regsub(req.http.Cookie, "^.*?frontend=([^;]*);*.*$", "\1"));
hash_data(regsub(req.http.Cookie, "^.*?frontend=([^;]*);*.*$", "\1"));
}
if (req.http.X - Varnish - Esi - Access == "customer_group" &&
req.http.Cookie~"customer_group=") {
hash_data(regsub(req.http.Cookie, "^.*?customer_group=([^;]*);*.*$", "\1"));
}
std.log("vcl_hash end return lookup");
return (lookup);
}
sub vcl_hit {}
sub vcl_backend_response {
set beresp.grace = 15 s;
set beresp.http.X - Varnish - Host = bereq.http.host;
set beresp.http.X - Varnish - URL = bereq.url;
if (bereq.url~"^(//media/|//skin/|/media/|/skin/|//js/|/js/|/)(?:(?:index|litespeed)\.php/)?") {
unset beresp.http.Vary;
set beresp.do_gzip = true;
if (beresp.status != 200 && beresp.status != 404) {
set beresp.ttl = 15 s;
set beresp.uncacheable = true;
return (deliver);
}
else {
if (beresp.http.Set - Cookie) {
set beresp.http.X - Varnish - Set - Cookie = beresp.http.Set - Cookie;
unset beresp.http.Set - Cookie;
}
unset beresp.http.Expires;
unset beresp.http.Pragma;
unset beresp.http.Cache;
unset beresp.http.Age;
if (beresp.http.X - Turpentine - Esi == "1") {
set beresp.do_esi = true;
}
if (beresp.http.X - Turpentine - Cache == "0") {
set beresp.ttl = 15 s;
set beresp.uncacheable = true;
set beresp.http.Cache - Control = "no-store, no-cache, must-revalidate";
return (deliver);
}
else {
if (true &&
bereq.url~".*\.(?:css|js|jpe?g|png|gif|ico|swf)(?=\?|&|$)") {
set beresp.ttl = 28800 s;
set beresp.http.Cache - Control = "max-age=28800";
}
elseif(bereq.http.X - Varnish - Esi - Method) {
if (bereq.http.X - Varnish - Esi - Access == "private" &&
bereq.http.Cookie~"frontend=") {
set beresp.http.X - Varnish - Session = regsub(bereq.http.Cookie,
"^.*?frontend=([^;]*);*.*$", "\1");
}
if (bereq.http.X - Varnish - Esi - Method == "ajax" &&
bereq.http.X - Varnish - Esi - Access == "public") {
set beresp.http.Cache - Control = "max-age=" + regsub(
bereq.url, ".*/ttl/(\d+)/.*", "\1");
}
set beresp.ttl = std.duration(
regsub(
bereq.url, ".*/ttl/(\d+)/.*", "\1s"),
300 s);
if (beresp.ttl == 0 s) {
set beresp.ttl = 15 s;
set beresp.uncacheable = true;
set beresp.http.Cache - Control = "no-store, no-cache, must-revalidate";
return (deliver);
}
} else {
set beresp.ttl = 3600 s;
}
}
}
return (deliver);
}
}
sub vcl_deliver {
if (req.http.X - Varnish - Faked - Session) {
set resp.http.Set - Cookie = req.http.X - Varnish - Faked - Session +
"; expires=" + resp.http.X - Varnish - Cookie - Expires + "; path=/";
if (req.http.Host) {
if (req.http.User - Agent~"^(?:ApacheBench/.*|.*Googlebot.*|.*APIs-Google.*|.*Mediapartners-Google.*|.*AdsBot-Google.*|JoeDog/.*|.*Siege/.*|magespeedtest\.com|Nexcessnet_Turpentine/.*|.*PTST.*|.*bingbot.*|.*msnbot.*|.*Baiduspider.*)$") {
set resp.http.Set - Cookie = resp.http.Set - Cookie +
"; domain=" + regsub(req.http.Host, ":\d+$", "");
}
else {
if (req.http.Host~"" && ""~"..") {
set resp.http.Set - Cookie = resp.http.Set - Cookie +
"; domain=";
}
else {
set resp.http.Set - Cookie = resp.http.Set - Cookie +
"; domain=" + regsub(req.http.Host, ":\d+$", "");
}
}
}
set resp.http.Set - Cookie = resp.http.Set - Cookie + "; httponly";
unset resp.http.X - Varnish - Cookie - Expires;
}
if (req.http.X - Varnish - Esi - Method == "ajax" && req.http.X - Varnish - Esi - Access == "private") {
set resp.http.Cache - Control = "no-cache";
}
if (false || client.ip~debug_acl) {
set resp.http.X - Varnish - Hits = obj.hits;
set resp.http.X - Varnish - Esi - Method = req.http.X - Varnish - Esi - Method;
set resp.http.X - Varnish - Esi - Access = req.http.X - Varnish - Esi - Access;
set resp.http.X - Varnish - Currency = req.http.X - Varnish - Currency;
set resp.http.X - Varnish - Store = req.http.X - Varnish - Store;
}
else {
unset resp.http.X - Varnish;
unset resp.http.Via;
unset resp.http.X - Powered - By;
unset resp.http.Server;
unset resp.http.X - Turpentine - Cache;
unset resp.http.X - Turpentine - Esi;
unset resp.http.X - Turpentine - Flush - Events;
unset resp.http.X - Turpentine - Block;
unset resp.http.X - Varnish - Session;
unset resp.http.X - Varnish - Host;
unset resp.http.X - Varnish - URL;
unset resp.http.X - Varnish - Set - Cookie;
}
}
Please do let me know if any other code is req
### My problem
I have successfully installed varnish and turpentine with my website and generated the above default.vcl file
Issue 1 => I am getteing js error almost in all pages
Issue 2 => My product page is getting a redirect to {{website_url}}:8080/{{my_product_page_url}} and then it gets connection error
I tried clearing the cache but not working
When exec below command gets following o/p
curl -I subdomain.example.com
HTTP/1.1 200 OK
Server: nginx/1.15.8
Date: Thu, 07 Feb 2019 13:20:08 GMT
Content-Type: text/html; charset=UTF-8
Vary: Accept-Encoding
Set-Cookie: frontend=f0hm6td2ekesdotlvdobqd9t4v; expires=Sun, 17-Feb-2019 19:26:48 GMT; Max-Age=886000; path=/; domain=subdomain.example.com; HttpOnly
Expires: Thu, 19 Nov 1981 08:52:00 GMT
Cache-Control: no-store, no-cache, must-revalidate
Pragma: no-cache
X-Frame-Options: SAMEORIGIN
X-Varnish: 2
Age: 0
Via: 1.1 varnish-v4
Accept-Ranges: bytes
Connection: keep-alive
When i save my varnish config and clear cache management and try to access the url in my browser.
getting below output
Accept-Ranges: bytes
Age: 0
Cache-Control: no-store, no-cache, must-revalidate
Connection: keep-alive
Content-Encoding: gzip
Content-Type: text/html; charset=UTF-8
Date: Thu, 07 Feb 2019 13:23:41 GMT
Server: nginx/1.15.8
Transfer-Encoding: chunked
Vary: Accept-Encoding
Via: 1.1 varnish-v4
X-Frame-Options: SAMEORIGIN
X-Turpentine-Cache: 1
X-Turpentine-Esi: 1
X-Varnish: 196655
X-Varnish-Currency:
X-Varnish-Esi-Access:
X-Varnish-Esi-Method:
X-Varnish-Hits: 0
X-Varnish-Host: subdomain.example.com
X-Varnish-Set-Cookie: frontend=cbr6qk9cg3ru50scmo7dmslt4p; expires=Sun, 17-Feb-2019 19:30:20 GMT; Max-Age=886000; path=/; domain=subdomain.example.com; HttpOnly
X-Varnish-Store:
X-Varnish-URL: /
Now the strange thing is that when i rerun the below command i cannot see X-Varnish
curl -I subdomain.example.com
HTTP/1.1 200 OK
Server: nginx/1.15.8
Date: Thu, 07 Feb 2019 13:27:24 GMT
Content-Type: text/html; charset=UTF-8
Connection: close
Vary: Accept-Encoding
Set-Cookie: frontend=l8fhhauteq896efe5k5o55ok8m; expires=Sun, 17-Feb-2019 19:34:03 GMT; Max-Age=886000; path=/; domain=subdomain.example.com; HttpOnly
Expires: Thu, 19 Nov 1981 08:52:00 GMT
Cache-Control: no-store, no-cache, must-revalidate
Pragma: no-cache
X-Frame-Options: SAMEORIGIN
I am pretty confused about the config part
Any help on this issue would be really helpfull.
miguelbalparda commented
This doesn't seem related to Turpentine but to your webserver config. I don't have much experience with nginx, all of our builds are using apache.
ChrisOrlando commented
A few issues I see here:
-
You have your nginx listening on port 80 but your varnish "backend" is configured as 127.0.0.1:8080
-
You have varnish set up to listen on port 80. This means varnish and nginx cannot possibly be running properly at the same time on your system since nginx and varnish cannot both bind to the port.
The first step would be to change your nginx configuration so it is serving content when requested at 127.0.0.1:8080 which is what varnish is using as a backend. You'll need to restart both services after doing so.
Kailash0593 commented
Hello Chris,
Thanks for pointing out the issue of the port.
I tried the above solution by changing nginx default port to 8080 and varnish to 80.
Restarted both the service.
But the issue still persists.
I tried the above solution by changing PHP v to 7.0. but still no luck.
I even tried by keeping the default nginx as a pure file without adding any other config.
Still the same issue is there.
I have read in the instructions guide to allow symlinks in Magento but still the issue is there.
I created the backup file of default.vcl file as etc/varnish/default_bckp.vcl and copied the default.vcl file generated by turpentine by below command
cp MATENTO_ROOT/var/default.vcl /etc/varnish/
Restarted both nginx and varnish.
I am using Amasty FPC extension which I disabled as mention in the doc to disable all the exsisting fpc to make turpentine work.
After trying all fix's I do clear the Magento cache.
Still the issue is there.