Initial Commit
This commit is contained in:
root
4
ngx_pagespeed-latest-beta/.gitignore
vendored
Normal file
4
ngx_pagespeed-latest-beta/.gitignore
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
test/tmp
|
||||
psol/
|
||||
psol-*.tar.gz
|
||||
*.*.*.*.tar.gz
|
||||
50
ngx_pagespeed-latest-beta/.travis.yml
Normal file
50
ngx_pagespeed-latest-beta/.travis.yml
Normal file
@@ -0,0 +1,50 @@
|
||||
language: c++
|
||||
install:
|
||||
- mv $TRAVIS_BUILD_DIR ~/ngxpagespeed
|
||||
- sudo apt-get install build-essential zlib1g-dev libpcre3 libpcre3-dev unzip g++ python gperf make devscripts fakeroot git curl netcat-traditional gcc-mozilla clang-3.4 2>&1 > /dev/null
|
||||
- export PATH=/usr/lib/gcc-mozilla/bin:$PATH
|
||||
- sudo ln -sf /usr/lib/gcc-mozilla/lib/libstdc++.so.6 /usr/lib/x86_64-linux-gnu/libstdc++.so.6
|
||||
# - sudo sh -c 'echo "image/webp webp" >> /etc/mime.types'
|
||||
# - mkdir -p ~/bin
|
||||
# - cd ~/bin
|
||||
# - git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git
|
||||
- mkdir ~/mod_pagespeed
|
||||
- cd ~/mod_pagespeed
|
||||
- git clone --recursive https://github.com/pagespeed/mod_pagespeed.git src
|
||||
- cd src
|
||||
- python build/gyp_chromium --depth=.
|
||||
- cd ~/mod_pagespeed/src/pagespeed/automatic
|
||||
- make BUILDTYPE=Release -C ../../pagespeed/automatic all
|
||||
- cd ~
|
||||
- git clone https://github.com/FRiCKLE/ngx_cache_purge.git
|
||||
- NGX_CACHE_PURGE=$PWD/ngx_cache_purge
|
||||
- wget https://openresty.org/download/ngx_openresty-1.9.7.2.tar.gz
|
||||
- tar xzf ngx_openresty-*.tar.gz
|
||||
- cd ngx_openresty-*/
|
||||
- ./configure --with-luajit
|
||||
- make
|
||||
- NGX_DEVEL_KIT=$(echo $HOME/ngx_openresty-*/build/ngx_devel_kit-*/)
|
||||
- SET_MISC_MODULE=$(echo $HOME/ngx_openresty-*/build/set-misc-nginx-module-*/)
|
||||
- HEADERS_MORE_MODULE=$(echo $HOME/ngx_openresty-*/build/headers-more-nginx-module-*/)
|
||||
- cd ~
|
||||
- wget https://github.com/nginx/nginx/archive/branches/default.zip
|
||||
- unzip default.zip
|
||||
- cd nginx-branches-default
|
||||
- MOD_PAGESPEED_DIR="$HOME/mod_pagespeed/src" ./auto/configure --add-module=$HOME/ngxpagespeed --add-module="$NGX_CACHE_PURGE" --add-module="$NGX_DEVEL_KIT" --add-module="$SET_MISC_MODULE" --add-module="$HEADERS_MORE_MODULE" --with-ipv6
|
||||
- make
|
||||
- sudo make install
|
||||
script:
|
||||
- echo "build successful"
|
||||
- echo "cd ~/ngxpagespeed"
|
||||
- echo "sudo ./test/run_tests.sh $HOME/mod_pagespeed $HOME/nginx-branches-default/objs/nginx"
|
||||
sudo: required
|
||||
compiler:
|
||||
- gcc
|
||||
notifications:
|
||||
email:
|
||||
- cheesy@google.com
|
||||
- jefftk@google.com
|
||||
- morlovich@google.com
|
||||
- jmarantz@google.com
|
||||
- huibao@google.com
|
||||
- jcrowell@google.com
|
||||
202
ngx_pagespeed-latest-beta/LICENSE
Normal file
202
ngx_pagespeed-latest-beta/LICENSE
Normal file
@@ -0,0 +1,202 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright 2010 Google Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
1
ngx_pagespeed-latest-beta/PSOL_BINARY_URL
Normal file
1
ngx_pagespeed-latest-beta/PSOL_BINARY_URL
Normal file
@@ -0,0 +1 @@
|
||||
https://dl.google.com/dl/page-speed/psol/1.12.34.2-$BIT_SIZE_NAME.tar.gz
|
||||
40
ngx_pagespeed-latest-beta/README.md
Normal file
40
ngx_pagespeed-latest-beta/README.md
Normal file
@@ -0,0 +1,40 @@
|
||||
|
||||
|
||||
[](https://travis-ci.org/pagespeed/ngx_pagespeed)
|
||||
|
||||
ngx_pagespeed speeds up your site and reduces page load time by automatically
|
||||
applying web performance best practices to pages and associated assets (CSS,
|
||||
JavaScript, images) without requiring you to modify your existing content or
|
||||
workflow. Features include:
|
||||
|
||||
- Image optimization: stripping meta-data, dynamic resizing, recompression
|
||||
- CSS & JavaScript minification, concatenation, inlining, and outlining
|
||||
- Small resource inlining
|
||||
- Deferring image and JavaScript loading
|
||||
- HTML rewriting
|
||||
- Cache lifetime extension
|
||||
- and
|
||||
[more](https://developers.google.com/speed/docs/mod_pagespeed/config_filters)
|
||||
|
||||
To see ngx_pagespeed in action, with example pages for each of the
|
||||
optimizations, see our <a href="http://ngxpagespeed.com">demonstration site</a>.
|
||||
|
||||
## How to build
|
||||
|
||||
Follow the steps on <a
|
||||
href="https://developers.google.com/speed/pagespeed/module/build_ngx_pagespeed_from_source">build
|
||||
ngx_pagespeed from source</a>.
|
||||
|
||||
## How to use
|
||||
|
||||
Follow the steps on <a
|
||||
href="https://developers.google.com/speed/pagespeed/module/configuration">PageSpeed
|
||||
configuration</a>.
|
||||
|
||||
For feedback, questions, and to follow
|
||||
the progress of the project:
|
||||
|
||||
- [ngx-pagespeed-discuss mailing
|
||||
list](https://groups.google.com/forum/#!forum/ngx-pagespeed-discuss)
|
||||
- [ngx-pagespeed-announce mailing
|
||||
list](https://groups.google.com/forum/#!forum/ngx-pagespeed-announce)
|
||||
384
ngx_pagespeed-latest-beta/config
Normal file
384
ngx_pagespeed-latest-beta/config
Normal file
@@ -0,0 +1,384 @@
|
||||
# Copyright 2012 Google Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
# Environment Variables (Optional):
|
||||
# MOD_PAGESPEED_DIR: absolute path to the mod_pagespeed/src directory
|
||||
# PSOL_BINARY: absolute path to pagespeed_automatic.a
|
||||
|
||||
mod_pagespeed_dir="${MOD_PAGESPEED_DIR:-unset}"
|
||||
position_aux="${POSITION_AUX:-unset}"
|
||||
|
||||
if [ "$mod_pagespeed_dir" = "unset" ] ; then
|
||||
mod_pagespeed_dir="$ngx_addon_dir/psol/include"
|
||||
build_from_source=false
|
||||
|
||||
if [ ! -e "$mod_pagespeed_dir" ] ; then
|
||||
echo "ngx_pagespeed: pagespeed optimization library not found:"
|
||||
|
||||
psol_binary_url="$($ngx_addon_dir/scripts/format_binary_url.sh \
|
||||
$ngx_addon_dir/PSOL_BINARY_URL)"
|
||||
if [[ "$psol_binary_url" != https://* ]]; then
|
||||
echo "
|
||||
This is a development branch of ngx_pagespeed, which means there is no
|
||||
precompiled PSOL library available to link against. Either build from a
|
||||
release tag, like latest-beta, or build PSOL from source:
|
||||
https://github.com/pagespeed/ngx_pagespeed/wiki/Building-PSOL-From-Source"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "
|
||||
You need to separately download the pagespeed library:
|
||||
$ cd $ngx_addon_dir
|
||||
$ wget $psol_binary_url
|
||||
$ tar -xzvf $(basename $psol_binary_url) # expands to psol/
|
||||
|
||||
Or see the installation instructions:
|
||||
https://developers.google.com/speed/pagespeed/module/build_ngx_pagespeed_from_source"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
build_from_source=true
|
||||
fi
|
||||
|
||||
os_name='unknown_os'
|
||||
arch_name='unknown_arch'
|
||||
uname_os=`uname`
|
||||
uname_arch=`uname -m`
|
||||
|
||||
if [ $uname_os = 'Linux' ]; then
|
||||
os_name='linux'
|
||||
elif [ $uname_os = 'Darwin' ]; then
|
||||
os_name='mac'
|
||||
else
|
||||
echo "OS not supported: $uname_os"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $uname_arch = 'x86_64' -o $uname_arch = 'amd64' ]; then
|
||||
arch_name='x64'
|
||||
elif [ $uname_arch = 'x86_32' -o $uname_arch = 'i686' \
|
||||
-o $uname_arch = 'i386' ]; then
|
||||
arch_name='ia32'
|
||||
else
|
||||
echo "Architecture not supported: $uname_arch"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$NGX_DEBUG" = "YES" ]; then
|
||||
buildtype=Debug
|
||||
# If we're using a psol tarball that doesn't contain Debug/ (which is the case
|
||||
# from 1.12 onward) then this will be overriden to buildtype=Release below.
|
||||
else
|
||||
buildtype=Release
|
||||
fi
|
||||
|
||||
# If the compiler is gcc, we want to use g++ to link, if at all possible,
|
||||
# so that -static-libstdc++ works.
|
||||
# Annoyingly, the feature test doesn't even use $LINK for linking, so that
|
||||
# needs an explicit -lstdc++
|
||||
pagespeed_libs=
|
||||
ps_maybe_gpp_base=`basename $CC| sed s/gcc/g++/`
|
||||
ps_maybe_gpp="`dirname $CC`/$ps_maybe_gpp_base"
|
||||
if [ -n "$NGX_GCC_VER" -a \( -x "$ps_maybe_gpp" \) ]; then
|
||||
LINK=$ps_maybe_gpp
|
||||
fi
|
||||
pagespeed_libs="-lstdc++"
|
||||
|
||||
# The compiler needs to know that __sync_add_and_fetch_4 is ok,
|
||||
# and this requires an instruction that didn't exist on i586 or i386.
|
||||
if [ "$uname_arch" = "i686" ]; then
|
||||
FLAG_MARCH='-march=i686'
|
||||
fi
|
||||
|
||||
CFLAGS="$CFLAGS $FLAG_MARCH"
|
||||
|
||||
# For now, standardize on gcc-4.x ABI --- if we don't set this, people building
|
||||
# with new gcc defaulting to gcc-5 C++11 ABI will have build trouble linking
|
||||
# to our libpsol.a
|
||||
# See https://gcc.gnu.org/onlinedocs/libstdc++/manual/using_dual_abi.html
|
||||
CFLAGS="$CFLAGS -D_GLIBCXX_USE_CXX11_ABI=0"
|
||||
CC_OLD_TEST_FLAGS="$CC_TEST_FLAGS"
|
||||
CC_TEST_FLAGS="$CC_TEST_FLAGS -D_GLIBCXX_USE_CXX11_ABI=0 --std=c++11"
|
||||
|
||||
case "$NGX_GCC_VER" in
|
||||
4.8*)
|
||||
# On GCC 4.8 and above, -Wall enables -Wunused-local-typedefs. This breaks
|
||||
# on VerifySizesAreEqual in bit_cast in chromium/src/base/basictypes.h which
|
||||
# has a typedef that is intentionally unused.
|
||||
CFLAGS="$CFLAGS -Wno-unused-local-typedefs"
|
||||
|
||||
# On GCC 4.8 and above, we get the following compiler warning:
|
||||
# chromium/src/base/memory/scoped_ptr.h:133:7: warning: declaration of ‘class scoped_ptr<C>’ [enabled by default]
|
||||
# Based on discussion at http://gcc.gnu.org/bugzilla/show_bug.cgi?id=54055,
|
||||
# this is invalid code, but hasn't been fixed yet in chromium.
|
||||
# Unfortunately, there also does not appear to be a flag for just disabling
|
||||
# that warning, so we add Wno-error to override nginx's default -Werror
|
||||
# option.
|
||||
CFLAGS="$CFLAGS -Wno-error"
|
||||
;;
|
||||
esac
|
||||
|
||||
# workaround for a bug in nginx-1.9.11, see:
|
||||
# http://hg.nginx.org/nginx/rev/ff1e625ae55b
|
||||
NGX_VERSION=`grep nginx_version src/core/nginx.h | sed -e 's/^.* \(.*\)$/\1/'`
|
||||
if [ "$NGX_VERSION" = "1009011" ]; then
|
||||
CFLAGS="$CFLAGS -Wno-write-strings"
|
||||
fi
|
||||
|
||||
if [ "$WNO_ERROR" = "YES" ]; then
|
||||
CFLAGS="$CFLAGS -Wno-error"
|
||||
fi
|
||||
|
||||
psol_binary="${PSOL_BINARY:-unset}"
|
||||
if [ "$psol_binary" = "unset" ] ; then
|
||||
if $build_from_source ; then
|
||||
psol_binary="\
|
||||
$mod_pagespeed_dir/pagespeed/automatic/pagespeed_automatic.a"
|
||||
else
|
||||
if ! [ -d "$ngx_addon_dir/psol/lib/$buildtype" ]; then
|
||||
echo "
|
||||
You have set --with-debug for building nginx, but precompiled Debug binaries for
|
||||
PSOL, which ngx_pagespeed depends on, aren't available. If you're trying to
|
||||
debug PSOL you need to build it from source. If you just want to run nginx with
|
||||
debug-level logging you can use the Release binaries."
|
||||
echo -n "
|
||||
Use the available Release binaries?"
|
||||
read -p " [Y/n] " yn
|
||||
if [[ "$yn" == N* || "$yn" == n* ]]; then
|
||||
echo "Cancelled."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
buildtype=Release
|
||||
fi
|
||||
psol_library_dir="$ngx_addon_dir/psol/lib/$buildtype/$os_name/$arch_name"
|
||||
psol_binary="$psol_library_dir/pagespeed_automatic.a"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "mod_pagespeed_dir=$mod_pagespeed_dir"
|
||||
echo "build_from_source=$build_from_source"
|
||||
|
||||
ngx_feature="psol"
|
||||
ngx_feature_name=""
|
||||
ngx_feature_run=no
|
||||
ngx_feature_incs="
|
||||
#include \"pagespeed/kernel/base/string.h\"
|
||||
#include \"pagespeed/kernel/base/string_writer.h\"
|
||||
#include \"pagespeed/kernel/base/null_message_handler.h\"
|
||||
#include \"pagespeed/kernel/html/html_parse.h\"
|
||||
#include \"pagespeed/kernel/html/html_writer_filter.h\"
|
||||
"
|
||||
|
||||
pagespeed_include="\
|
||||
$mod_pagespeed_dir \
|
||||
$mod_pagespeed_dir/third_party/chromium/src \
|
||||
$mod_pagespeed_dir/third_party/google-sparsehash/src/src \
|
||||
$mod_pagespeed_dir/third_party/google-sparsehash/gen/arch/$os_name/$arch_name/include \
|
||||
$mod_pagespeed_dir/third_party/grpc/src/include \
|
||||
$mod_pagespeed_dir/third_party/protobuf/src/src \
|
||||
$mod_pagespeed_dir/third_party/re2/src \
|
||||
$mod_pagespeed_dir/out/$buildtype/obj/gen \
|
||||
$mod_pagespeed_dir/out/$buildtype/obj/gen/protoc_out/instaweb \
|
||||
$mod_pagespeed_dir/third_party/apr/src/include \
|
||||
$mod_pagespeed_dir/third_party/aprutil/src/include \
|
||||
$mod_pagespeed_dir/third_party/apr/gen/arch/$os_name/$arch_name/include \
|
||||
$mod_pagespeed_dir/third_party/aprutil/gen/arch/$os_name/$arch_name/include \
|
||||
$mod_pagespeed_dir/url"
|
||||
ngx_feature_path="$pagespeed_include"
|
||||
|
||||
pagespeed_libs="$psol_binary $pagespeed_libs -lrt -pthread -lm"
|
||||
ngx_feature_libs="$pagespeed_libs"
|
||||
ngx_feature_test="
|
||||
|
||||
GoogleString output_buffer;
|
||||
net_instaweb::StringWriter write_to_string(&output_buffer);
|
||||
|
||||
net_instaweb::NullMessageHandler handler;
|
||||
net_instaweb::HtmlParse html_parse(&handler);
|
||||
net_instaweb::HtmlWriterFilter html_writer_filter(&html_parse);
|
||||
|
||||
html_writer_filter.set_writer(&write_to_string);
|
||||
html_parse.AddFilter(&html_writer_filter);
|
||||
|
||||
html_parse.StartParse(\"http:example.com\");
|
||||
html_parse.ParseText(
|
||||
\"<html ><body ><h1 >Test</h1 ><p>Test Text</p></body></html>\n\");
|
||||
html_parse.FinishParse();
|
||||
|
||||
printf(\"parsed as: %s\", output_buffer.c_str())"
|
||||
|
||||
# Test whether we have pagespeed and can compile and link against it.
|
||||
. "$ngx_addon_dir/cpp_feature"
|
||||
|
||||
if [ $ngx_found = no ]; then
|
||||
cat << END
|
||||
$0: error: module ngx_pagespeed requires the pagespeed optimization library.
|
||||
Look in $PWD/$NGX_AUTOCONF_ERR for more details.
|
||||
END
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ps_src="$ngx_addon_dir/src"
|
||||
ngx_addon_name=ngx_pagespeed
|
||||
NGX_ADDON_DEPS="$NGX_ADDON_DEPS \
|
||||
$ps_src/log_message_handler.h \
|
||||
$ps_src/ngx_base_fetch.h \
|
||||
$ps_src/ngx_caching_headers.h \
|
||||
$ps_src/ngx_event_connection.h \
|
||||
$ps_src/ngx_fetch.h \
|
||||
$ps_src/ngx_gzip_setter.h \
|
||||
$ps_src/ngx_list_iterator.h \
|
||||
$ps_src/ngx_message_handler.h \
|
||||
$ps_src/ngx_pagespeed.h \
|
||||
$ps_src/ngx_rewrite_driver_factory.h \
|
||||
$ps_src/ngx_rewrite_options.h \
|
||||
$ps_src/ngx_server_context.h \
|
||||
$ps_src/ngx_url_async_fetcher.h \
|
||||
$psol_binary"
|
||||
NPS_SRCS=" \
|
||||
$ps_src/log_message_handler.cc \
|
||||
$ps_src/ngx_base_fetch.cc \
|
||||
$ps_src/ngx_caching_headers.cc \
|
||||
$ps_src/ngx_event_connection.cc \
|
||||
$ps_src/ngx_fetch.cc \
|
||||
$ps_src/ngx_gzip_setter.cc \
|
||||
$ps_src/ngx_list_iterator.cc \
|
||||
$ps_src/ngx_message_handler.cc \
|
||||
$ps_src/ngx_pagespeed.cc \
|
||||
$ps_src/ngx_rewrite_driver_factory.cc \
|
||||
$ps_src/ngx_rewrite_options.cc \
|
||||
$ps_src/ngx_server_context.cc \
|
||||
$ps_src/ngx_url_async_fetcher.cc"
|
||||
# Save our sources in a separate var since we may need it in config.make
|
||||
PS_NGX_SRCS="$NGX_ADDON_SRCS \
|
||||
$NPS_SRCS"
|
||||
|
||||
# Make pagespeed run immediately before gzip and Brotli.
|
||||
if echo $HTTP_FILTER_MODULES | grep ngx_http_brotli_filter_module >/dev/null; then
|
||||
next=ngx_http_brotli_filter_module
|
||||
elif [ $HTTP_GZIP = YES ]; then
|
||||
next=ngx_http_gzip_filter_module
|
||||
else
|
||||
next=ngx_http_range_header_filter_module
|
||||
fi
|
||||
|
||||
if [ -n "$ngx_module_link" ]; then
|
||||
# nginx-1.9.11+
|
||||
ngx_module_type=HTTP_FILTER
|
||||
ngx_module_name="ngx_pagespeed ngx_pagespeed_etag_filter"
|
||||
ngx_module_incs="$ngx_feature_path"
|
||||
ngx_module_deps=
|
||||
ngx_module_srcs="$NPS_SRCS"
|
||||
ngx_module_libs="$ngx_feature_libs"
|
||||
ngx_module_order="ngx_http_range_header_filter_module\
|
||||
ngx_pagespeed_etag_filter\
|
||||
ngx_http_gzip_filter_module \
|
||||
ngx_http_brotli_filter_module \
|
||||
ngx_pagespeed \
|
||||
ngx_http_postpone_filter_module \
|
||||
ngx_http_ssi_filter_module \
|
||||
ngx_http_charset_filter_module \
|
||||
ngx_http_xslt_filter_module \
|
||||
ngx_http_image_filter_module \
|
||||
ngx_http_sub_filter_module \
|
||||
ngx_http_addition_filter_module \
|
||||
ngx_http_gunzip_filter_module \
|
||||
ngx_http_userid_filter_module \
|
||||
ngx_http_headers_filter_module"
|
||||
|
||||
. auto/module
|
||||
|
||||
if [ $ngx_module_link != DYNAMIC ]; then
|
||||
# ngx_module_order doesn't work with static modules,
|
||||
# so we must re-order filters here.
|
||||
if [ "$position_aux" = "true" ] ; then
|
||||
HTTP_AUX_FILTER_MODULES="$HTTP_AUX_FILTER_MODULES $ngx_addon_name"
|
||||
else
|
||||
HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES \
|
||||
| sed "s/ngx_pagespeed//" \
|
||||
| sed "s/$next/$next ngx_pagespeed/")
|
||||
fi
|
||||
# Make the etag header filter run immediately before range header filter.
|
||||
HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES \
|
||||
| sed "s/ngx_pagespeed_etag_filter//" \
|
||||
| sed "s/ngx_http_range_header_filter_module/ngx_http_range_header_filter_module ngx_pagespeed_etag_filter/")
|
||||
else
|
||||
if [ "$position_aux" = "true" ] ; then
|
||||
ngx_module_type=HTTP_AUX_FILTER
|
||||
ngx_module_order=""
|
||||
fi
|
||||
fi
|
||||
else
|
||||
CORE_LIBS="$CORE_LIBS $pagespeed_libs"
|
||||
CORE_INCS="$CORE_INCS $pagespeed_include"
|
||||
NGX_ADDON_SRCS="$PS_NGX_SRCS"
|
||||
if [ "$position_aux" = "true" ] ; then
|
||||
HTTP_AUX_FILTER_MODULES="$HTTP_AUX_FILTER_MODULES $ngx_addon_name"
|
||||
else
|
||||
HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES | sed "s/$next/$next $ngx_addon_name/")
|
||||
fi
|
||||
|
||||
# Make the etag header filter run immediately before range header filter.
|
||||
HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES |\
|
||||
sed "s/ngx_http_range_header_filter_module/ngx_http_range_header_filter_module ngx_pagespeed_etag_filter/")
|
||||
fi
|
||||
|
||||
echo "List of modules (in reverse order of applicability): "$HTTP_FILTER_MODULES
|
||||
|
||||
# Test whether the compiler is compatible
|
||||
ngx_feature="psol-compiler-compat"
|
||||
ngx_feature_name=""
|
||||
ngx_feature_run=no
|
||||
ngx_feature_incs=""
|
||||
ngx_feature_path=""
|
||||
ngx_feature_libs="-lstdc++"
|
||||
ngx_feature_test="
|
||||
|
||||
#if defined(__clang__) && defined(__GLIBCXX__)
|
||||
// See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html#abi.versioning
|
||||
// for a list of various values of __GLIBCXX__. Note that they're not monotonic
|
||||
// with respect to version numbers.
|
||||
#if __GLIBCXX__ == 20120322 || __GLIBCXX__ == 20120614
|
||||
#error \"clang is using libstdc++ 4.7.0 or 4.7.1, which can cause binary incompatibility.\"
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#if !defined(__clang__) && defined(__GNUC__)
|
||||
#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 8)
|
||||
#error \"GCC < 4.8 no longer supported. Please use gcc >= 4.8 or clang >= 3.3\"
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#if defined(__clang__)
|
||||
#if __clang_major__ < 3 || (__clang_major__ == 3 && __clang_minor__ < 3)
|
||||
#error \"Please use gcc >= 4.8 or clang >= 3.3\"
|
||||
#endif
|
||||
#endif
|
||||
"
|
||||
|
||||
. "$ngx_addon_dir/cpp_feature"
|
||||
|
||||
if [ $ngx_found = no ]; then
|
||||
cat << END
|
||||
$0: error: module ngx_pagespeed requires gcc >= 4.8 or clang >= 3.3.
|
||||
See https://developers.google.com/speed/pagespeed/module/build_ngx_pagespeed_from_source for some recommendations.
|
||||
Look in $PWD/$NGX_AUTOCONF_ERR for more details.
|
||||
END
|
||||
exit 1
|
||||
fi
|
||||
|
||||
have=NGX_PAGESPEED . auto/have
|
||||
|
||||
CC_TEST_FLAGS="$CC_OLD_TEST_FLAGS"
|
||||
7
ngx_pagespeed-latest-beta/config.make
Normal file
7
ngx_pagespeed-latest-beta/config.make
Normal file
@@ -0,0 +1,7 @@
|
||||
# Since nginx build system doesn't normally do C++, there is no CXXFLAGS for us
|
||||
# to touch, and compilers are understandably unhappy with --std=c++11 on C
|
||||
# files. Hence, we hack the makefile to add it for just our sources.
|
||||
for ps_src_file in $PS_NGX_SRCS; do
|
||||
ps_obj_file="$NGX_OBJS/addon/src/`basename $ps_src_file .cc`.o"
|
||||
echo "$ps_obj_file : CFLAGS += --std=c++11" >> $NGX_MAKEFILE
|
||||
done
|
||||
122
ngx_pagespeed-latest-beta/cpp_feature
Normal file
122
ngx_pagespeed-latest-beta/cpp_feature
Normal file
@@ -0,0 +1,122 @@
|
||||
# Copyright (C) Igor Sysoev
|
||||
# Copyright (C) Nginx, Inc.
|
||||
# 2012-10-01 Modified from auto/feature by jefftk to support c++ test files.
|
||||
|
||||
echo $ngx_n "checking for $ngx_feature ...$ngx_c"
|
||||
|
||||
cat << END >> $NGX_AUTOCONF_ERR
|
||||
|
||||
----------------------------------------
|
||||
checking for $ngx_feature
|
||||
|
||||
END
|
||||
|
||||
ngx_found=no
|
||||
|
||||
if test -n "$ngx_feature_name"; then
|
||||
ngx_have_feature=`echo $ngx_feature_name \
|
||||
| tr abcdefghijklmnopqrstuvwxyz ABCDEFGHIJKLMNOPQRSTUVWXYZ`
|
||||
fi
|
||||
|
||||
if test -n "$ngx_feature_path"; then
|
||||
for ngx_temp in $ngx_feature_path; do
|
||||
ngx_feature_inc_path="$ngx_feature_inc_path -I $ngx_temp"
|
||||
done
|
||||
fi
|
||||
|
||||
cat << END > $NGX_AUTOTEST.cc
|
||||
|
||||
#include <sys/types.h>
|
||||
$NGX_INCLUDE_UNISTD_H
|
||||
$ngx_feature_incs
|
||||
|
||||
int main() {
|
||||
$ngx_feature_test;
|
||||
return 0;
|
||||
}
|
||||
|
||||
END
|
||||
|
||||
|
||||
ngx_test="$CC $CC_TEST_FLAGS $CC_AUX_FLAGS $ngx_feature_inc_path \
|
||||
-o $NGX_AUTOTEST $NGX_AUTOTEST.cc $NGX_TEST_LD_OPT $ngx_feature_libs"
|
||||
|
||||
ngx_feature_inc_path=
|
||||
|
||||
eval "/bin/sh -c \"$ngx_test\" >> $NGX_AUTOCONF_ERR 2>&1"
|
||||
|
||||
|
||||
if [ -x $NGX_AUTOTEST ]; then
|
||||
|
||||
case "$ngx_feature_run" in
|
||||
|
||||
yes)
|
||||
# /bin/sh is used to intercept "Killed" or "Abort trap" messages
|
||||
if /bin/sh -c $NGX_AUTOTEST >> $NGX_AUTOCONF_ERR 2>&1; then
|
||||
echo " found"
|
||||
ngx_found=yes
|
||||
|
||||
if test -n "$ngx_feature_name"; then
|
||||
have=$ngx_have_feature . auto/have
|
||||
fi
|
||||
|
||||
else
|
||||
echo " found but is not working"
|
||||
fi
|
||||
;;
|
||||
|
||||
value)
|
||||
# /bin/sh is used to intercept "Killed" or "Abort trap" messages
|
||||
if /bin/sh -c $NGX_AUTOTEST >> $NGX_AUTOCONF_ERR 2>&1; then
|
||||
echo " found"
|
||||
ngx_found=yes
|
||||
|
||||
cat << END >> $NGX_AUTO_CONFIG_H
|
||||
|
||||
#ifndef $ngx_feature_name
|
||||
#define $ngx_feature_name `$NGX_AUTOTEST`
|
||||
#endif
|
||||
|
||||
END
|
||||
else
|
||||
echo " found but is not working"
|
||||
fi
|
||||
;;
|
||||
|
||||
bug)
|
||||
# /bin/sh is used to intercept "Killed" or "Abort trap" messages
|
||||
if /bin/sh -c $NGX_AUTOTEST >> $NGX_AUTOCONF_ERR 2>&1; then
|
||||
echo " not found"
|
||||
|
||||
else
|
||||
echo " found"
|
||||
ngx_found=yes
|
||||
|
||||
if test -n "$ngx_feature_name"; then
|
||||
have=$ngx_have_feature . auto/have
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
|
||||
*)
|
||||
echo " found"
|
||||
ngx_found=yes
|
||||
|
||||
if test -n "$ngx_feature_name"; then
|
||||
have=$ngx_have_feature . auto/have
|
||||
fi
|
||||
;;
|
||||
|
||||
esac
|
||||
|
||||
else
|
||||
echo " not found"
|
||||
|
||||
echo "----------" >> $NGX_AUTOCONF_ERR
|
||||
cat $NGX_AUTOTEST.cc >> $NGX_AUTOCONF_ERR
|
||||
echo "----------" >> $NGX_AUTOCONF_ERR
|
||||
echo $ngx_test >> $NGX_AUTOCONF_ERR
|
||||
echo "----------" >> $NGX_AUTOCONF_ERR
|
||||
fi
|
||||
|
||||
rm $NGX_AUTOTEST*
|
||||
534
ngx_pagespeed-latest-beta/scripts/build_ngx_pagespeed.sh
Executable file
534
ngx_pagespeed-latest-beta/scripts/build_ngx_pagespeed.sh
Executable file
@@ -0,0 +1,534 @@
|
||||
#!/bin/bash
|
||||
|
||||
function usage() {
|
||||
echo "
|
||||
Usage: build_ngx_pagespeed.sh [options]
|
||||
|
||||
Installs ngx_pagespeed and its dependencies. Can optionally build and install
|
||||
nginx as well.
|
||||
|
||||
Options:
|
||||
-v, --ngx-pagespeed-version <ngx_pagespeed version>
|
||||
What version of ngx_pagespeed to build. Valid options include:
|
||||
* latest-beta
|
||||
* latest-stable
|
||||
* a version number, such as 1.11.33.4
|
||||
|
||||
If you don't specify a version, defaults to latest-stable.
|
||||
|
||||
-n, --nginx-version <nginx version>
|
||||
What version of nginx to build. If not set, this script only prepares the
|
||||
ngx_pagespeed module, and expects you to handle including it when you
|
||||
build nginx.
|
||||
|
||||
If you pass in 'latest' then this script scrapes the nginx download page
|
||||
and attempts to determine the latest version automatically.
|
||||
|
||||
-m, --dynamic-module
|
||||
Build ngx_pagespeed as a dynamic module.
|
||||
|
||||
-b, --builddir <directory>
|
||||
Where to build. Defaults to \$HOME.
|
||||
|
||||
-p, --no-deps-check
|
||||
By default, this script checks for the packages it depends on and tries to
|
||||
install them. If you have installed dependencies from source or are on a
|
||||
non-deb non-rpm system, this won't work. In that case, install the
|
||||
dependencies yourself and pass --no-deps-check.
|
||||
|
||||
-d, --dryrun
|
||||
Don't make any changes to the system, just print what changes you
|
||||
would have made.
|
||||
|
||||
-h, --help
|
||||
Print this message and exit."
|
||||
}
|
||||
|
||||
# Prints an error message and exits with an error code.
|
||||
function fail() {
|
||||
local error_message="$@"
|
||||
echo "$@" >&2
|
||||
|
||||
# Normally I'd use $0 in "usage" here, but since most people will be running
|
||||
# this via curl, that wouldn't actually give something useful.
|
||||
echo >&2
|
||||
echo "For usage information, run this script with --help" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Intended to be called as:
|
||||
# bash <(curl dl.google.com/.../build_ngx_pagespeed.sh) <args>
|
||||
|
||||
# If we set -e or -u then users of this script will see it silently exit on
|
||||
# failure. Instead we need to check the exit status of each command manually.
|
||||
# The run function handles exit-status checking for system-changing commands.
|
||||
# Additionally, this allows us to easily have a dryrun mode where we don't
|
||||
# actually make any changes.
|
||||
function run() {
|
||||
if "$DRYRUN"; then
|
||||
echo "would run $@"
|
||||
else
|
||||
if ! "$@"; then
|
||||
echo "Failure running $@, exiting."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
function redhat_is_installed() {
|
||||
local package_name="$1"
|
||||
rpm -qa $package_name | grep -q .
|
||||
}
|
||||
|
||||
function debian_is_installed() {
|
||||
local package_name="$1"
|
||||
dpkg -l $package_name | grep ^ii | grep -q .
|
||||
}
|
||||
|
||||
function version_sort() {
|
||||
# We'd rather use sort -V, but that's not available on Centos 5. This works
|
||||
# for versions in the form A.B.C.D or shorter, which is enough for our use.
|
||||
sort -t '.' -k 1,1 -k 2,2 -k 3,3 -k 4,4 -g
|
||||
}
|
||||
|
||||
# Compare two numeric versions in the form "A.B.C". Works with version numbers
|
||||
# having up to four components, since that's enough to handle both nginx (3) and
|
||||
# ngx_pagespeed (4).
|
||||
function version_older_than() {
|
||||
local test_version="$1"
|
||||
local compare_to="$2"
|
||||
|
||||
local older_version=$(echo $@ | tr ' ' '\n' | version_sort | head -n 1)
|
||||
test "$older_version" != "$compare_to"
|
||||
}
|
||||
|
||||
function determine_latest_nginx_version() {
|
||||
# Scrape nginx's download page to try to find the most recent nginx version.
|
||||
|
||||
nginx_download_url="https://nginx.org/en/download.html"
|
||||
function report_error() {
|
||||
fail "
|
||||
Couldn't automatically determine the latest nginx version: failed to $@
|
||||
$nginx_download_url"
|
||||
}
|
||||
|
||||
nginx_download_page=$(curl -sS --fail "$nginx_download_url") || \
|
||||
report_error "download"
|
||||
|
||||
download_refs=$(echo "$nginx_download_page" | \
|
||||
grep -o '/download/nginx-[0-9.]*[.]tar[.]gz') || \
|
||||
report_error "parse"
|
||||
|
||||
versions_available=$(echo "$download_refs" | \
|
||||
sed -e 's~^/download/nginx-~~' -e 's~\.tar\.gz$~~') || \
|
||||
report_error "extract versions from"
|
||||
|
||||
latest_version=$(echo "$versions_available" | version_sort | tail -n 1) || \
|
||||
report_error "determine latest version from"
|
||||
|
||||
if version_older_than "$latest_version" "1.11.4"; then
|
||||
fail "
|
||||
Expected the latest version of nginx to be at least 1.11.4 but found
|
||||
$latest_version on $nginx_download_url"
|
||||
fi
|
||||
|
||||
echo "$latest_version"
|
||||
}
|
||||
|
||||
# Usage:
|
||||
# install_dependencies install_pkg_cmd is_pkg_installed_cmd dep1 dep2 ...
|
||||
#
|
||||
# install_pkg_cmd is a command to install a dependency
|
||||
# is_pkg_installed_cmd is a command that returns true if the dependency is
|
||||
# already installed
|
||||
# each dependency is a package name
|
||||
function install_dependencies() {
|
||||
local install_pkg_cmd="$1"
|
||||
local is_pkg_installed_cmd="$2"
|
||||
shift 2
|
||||
|
||||
local missing_dependencies=""
|
||||
|
||||
for package_name in "$@"; do
|
||||
if ! $is_pkg_installed_cmd $package_name; then
|
||||
missing_dependencies+="$package_name "
|
||||
fi
|
||||
done
|
||||
if [ -n "$missing_dependencies" ]; then
|
||||
echo "Detected that we're missing the following depencencies:"
|
||||
echo " $missing_dependencies"
|
||||
echo "Installing them:"
|
||||
run sudo $install_pkg_cmd $missing_dependencies
|
||||
fi
|
||||
}
|
||||
|
||||
function gcc_too_old() {
|
||||
# We need gcc >= 4.8
|
||||
local gcc_major_version=$(gcc -dumpversion | awk -F. '{print $1}')
|
||||
if [ "$gcc_major_version" -lt 4 ]; then
|
||||
return 0 # too old
|
||||
elif [ "$gcc_major_version" -gt 4 ]; then
|
||||
return 1 # plenty new
|
||||
fi
|
||||
# It's gcc 4.x, check if x >= 8:
|
||||
local gcc_minor_version=$(gcc -dumpversion | awk -F. '{print $2}')
|
||||
test "$gcc_minor_version" -lt 8
|
||||
}
|
||||
|
||||
function continue_or_exit() {
|
||||
local prompt="$1"
|
||||
read -p "$prompt [Y/n] " yn
|
||||
if [[ "$yn" == N* || "$yn" == n* ]]; then
|
||||
echo "Cancelled."
|
||||
exit 0
|
||||
fi
|
||||
}
|
||||
|
||||
# If a string is very simple we don't need to quote it. But we should quote
|
||||
# everything else to be safe.
|
||||
function needs_quoting() {
|
||||
echo "$@" | grep -q '[^a-zA-Z0-9./_=-]'
|
||||
}
|
||||
|
||||
function escape_for_quotes() {
|
||||
echo "$@" | sed -e 's~\\~\\\\~g' -e "s~'~\\\\'~g"
|
||||
}
|
||||
|
||||
function quote_arguments() {
|
||||
local argument_str=""
|
||||
for argument in "$@"; do
|
||||
if [ -n "$argument_str" ]; then
|
||||
argument_str+=" "
|
||||
fi
|
||||
if needs_quoting "$argument"; then
|
||||
argument="'$(escape_for_quotes "$argument")'"
|
||||
fi
|
||||
argument_str+="$argument"
|
||||
done
|
||||
echo "$argument_str"
|
||||
}
|
||||
|
||||
function build_ngx_pagespeed() {
|
||||
getopt --test
|
||||
if [ "$?" != 4 ]; then
|
||||
# Even Centos 5 and Ubuntu 10 LTS have new-style getopt, so I don't expect
|
||||
# this to be hit in practice on systems that are actually able to run
|
||||
# PageSpeed.
|
||||
fail "Your version of getopt is too old. Exiting with no changes made."
|
||||
fi
|
||||
|
||||
opts=$(getopt -o v:n:mb:pdh \
|
||||
--longoptions ngx-pagespeed-version:,nginx-version:,dynamic-module \
|
||||
--longoptions buildir:,no-deps-check,dryrun,help \
|
||||
-n "$(basename "$0")" -- "$@")
|
||||
if [ $? != 0 ]; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
eval set -- "$opts"
|
||||
|
||||
NPS_VERSION="latest-stable"
|
||||
NGINX_VERSION=""
|
||||
BUILDDIR="$HOME"
|
||||
DO_DEPS_CHECK=true
|
||||
DRYRUN=false
|
||||
DYNAMIC_MODULE=false
|
||||
while true; do
|
||||
case "$1" in
|
||||
-v | --ngx-pagespeed-version) shift
|
||||
NPS_VERSION="$1"
|
||||
shift
|
||||
;;
|
||||
-n | --nginx-version) shift
|
||||
NGINX_VERSION="$1"
|
||||
shift
|
||||
;;
|
||||
-m | --dynamic-module) shift
|
||||
DYNAMIC_MODULE=true
|
||||
;;
|
||||
-b | --builddir) shift
|
||||
BUILDDIR="$1"
|
||||
shift
|
||||
;;
|
||||
-p | --no-deps-check) shift
|
||||
DO_DEPS_CHECK=false
|
||||
;;
|
||||
-d | --dryrun) shift
|
||||
DRYRUN="true"
|
||||
;;
|
||||
-h | --help) shift
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
--) shift
|
||||
break
|
||||
;;
|
||||
*)
|
||||
echo "Invalid argument: $1"
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ ! -d "$BUILDDIR" ]; then
|
||||
fail "Told to build in $BUILDDIR, but that directory doesn't exist."
|
||||
fi
|
||||
|
||||
if [ "$NGINX_VERSION" = "latest" ]; then
|
||||
# When this function fails it prints the debugging information needed first
|
||||
# to stderr.
|
||||
NGINX_VERSION=$(determine_latest_nginx_version) || exit 1
|
||||
fi
|
||||
|
||||
if "$DYNAMIC_MODULE"; then
|
||||
# Check that ngx_pagespeed and nginx are recent enough to support dynamic
|
||||
# modules.
|
||||
if version_older_than "$NPS_VERSION" "1.10.33.5"; then
|
||||
fail "
|
||||
You're trying to build ngx_pagespeed $NPS_VERSION as a dynamic module, but
|
||||
ngx_pagespeed didn't add support for dynamic modules until 1.10.33.5."
|
||||
fi
|
||||
|
||||
if [ ! -z "NGINX_VERSION" ]; then
|
||||
if version_older_than "$NGINX_VERSION" "1.9.13"; then
|
||||
fail "
|
||||
You're trying to build nginx $NGINX_VERSION as a dynamic module but nginx didn't
|
||||
add support for dynamic modules in a way compatible with ngx_pagespeed until
|
||||
1.9.13."
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if "$DRYRUN"; then
|
||||
TEMPDIR="/tmp/output-of-mktemp"
|
||||
else
|
||||
TEMPDIR=$(mktemp -d)
|
||||
function cleanup_tempdir {
|
||||
rm -rf "$TEMPDIR"
|
||||
}
|
||||
trap cleanup_tempdir EXIT
|
||||
fi
|
||||
|
||||
extra_flags=()
|
||||
# Now make sure our dependencies are installed.
|
||||
if "$DO_DEPS_CHECK"; then
|
||||
if [ -f /etc/debian_version ]; then
|
||||
echo "Detected debian-based distro."
|
||||
|
||||
install_dependencies "apt-get install" debian_is_installed \
|
||||
"build-essential zlib1g-dev libpcre3 libpcre3-dev unzip"
|
||||
|
||||
if gcc_too_old; then
|
||||
if [ ! -e /usr/lib/gcc-mozilla/bin/gcc ]; then
|
||||
echo "Detected that gcc is older than 4.8. Installing gcc-mozilla"
|
||||
echo "which installs gcc-4.8 into /usr/lib/gcc-mozilla/ and doesn't"
|
||||
echo "affect your global gcc installation."
|
||||
run sudo apt-get install gcc-mozilla
|
||||
fi
|
||||
|
||||
extra_flags=("--with-cc=/usr/lib/gcc-mozilla/bin/gcc" \
|
||||
"--with-ld-opt=-static-libstdc++")
|
||||
fi
|
||||
|
||||
elif [ -f /etc/redhat-release ]; then
|
||||
echo "Detected redhat-based distro."
|
||||
|
||||
install_dependencies "yum install" redhat_is_installed \
|
||||
"gcc-c++ pcre-devel zlib-devel make unzip wget"
|
||||
if gcc_too_old; then
|
||||
if [ ! -e /opt/rh/devtoolset-2/root/usr/bin/gcc ]; then
|
||||
redhat_major_version=$(
|
||||
cat /etc/redhat-release | grep -o -E '[0-9]+' | head -n 1)
|
||||
if [ "$redhat_major_version" == 5 ]; then
|
||||
slc_version=5
|
||||
elif [ "$redhat_major_version" == 6 ]; then
|
||||
slc_version=6
|
||||
else
|
||||
fail "
|
||||
Unexpected major version $redhat_major_version in /etc/redhat-release:
|
||||
$(cat /etc/redhat-release) Expected 5 or 6."
|
||||
fi
|
||||
|
||||
echo "Detected that gcc is older than 4.8. Scientific Linux provides"
|
||||
echo "a gcc package that installs gcc-4.8 into /opt/ and doesn't"
|
||||
echo "affect your global gcc installation."
|
||||
slc_key="https://linux.web.cern.ch/linux/scientific6/docs/repository/"
|
||||
slc_key+="cern/slc6X/i386/RPM-GPG-KEY-cern"
|
||||
slc_key_out="$TEMPDIR/RPM-GPG-KEY-cern"
|
||||
run sudo wget "$slc_key" -O "$slc_key_out"
|
||||
run sudo rpm --import "$slc_key_out"
|
||||
|
||||
repo_fname="/etc/yum.repos.d/slc${slc_version}-devtoolset.repo"
|
||||
if [ -e "$repo_fname" ]; then
|
||||
fail "Expected $repo_fname not to exist; aborting."
|
||||
fi
|
||||
|
||||
repo_url="https://linux.web.cern.ch/linux/scientific${slc_version}/"
|
||||
repo_url+="/docs/repository/cern/devtoolset/"
|
||||
repo_url+="slc${slc_version}-devtoolset.repo"
|
||||
run sudo wget -O "$repo_fname" "$repo_url"
|
||||
run sudo yum install devtoolset-2-gcc-c++ devtoolset-2-binutils
|
||||
fi
|
||||
extra_flags=("--with-cc=/opt/rh/devtoolset-2/root/usr/bin/gcc")
|
||||
fi
|
||||
else
|
||||
fail "
|
||||
This doesn't appear to be a deb-based distro or an rpm-based one. Not going to
|
||||
be able to install dependencies. Please install dependencies manually and rerun
|
||||
with --no-deps-check."
|
||||
fi
|
||||
echo "Dependencies are all set."
|
||||
else
|
||||
echo "Not checking whether dependencies are installed."
|
||||
fi
|
||||
|
||||
function delete_if_already_exists() {
|
||||
if "$DRYRUN"; then return; fi
|
||||
|
||||
local directory="$1"
|
||||
if [ -d "$directory" ]; then
|
||||
if [ ${#directory} -lt 8 ]; then
|
||||
fail "
|
||||
Not deleting $directory; name is suspiciously short. Something is wrong."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
continue_or_exit "OK to delete $directory?"
|
||||
run rm -rf "$directory"
|
||||
fi
|
||||
}
|
||||
|
||||
nps_baseurl="https://github.com/pagespeed/ngx_pagespeed/archive"
|
||||
# In general, the zip github builds for tag foo unzips to ngx_pagespeed-foo,
|
||||
# but it looks like they special case vVERSION tags to ngx_pagespeed-VERSION.
|
||||
if [[ "$NPS_VERSION" =~ ^[0-9]*[.][0-9]*[.][0-9]*[.][0-9]*$ ]]; then
|
||||
# We've been given a numeric version number. This has an associated tag in
|
||||
# the form vVERSION-beta.
|
||||
nps_url_fname="v${NPS_VERSION}-beta"
|
||||
nps_downloaded_fname="ngx_pagespeed-${NPS_VERSION}-beta"
|
||||
else
|
||||
# We've been given a tag name, like latest-beta. Download that directly.
|
||||
nps_url_fname="$NPS_VERSION"
|
||||
nps_downloaded_fname="ngx_pagespeed-${NPS_VERSION}"
|
||||
fi
|
||||
|
||||
nps_downloaded="$TEMPDIR/$nps_downloaded_fname.zip"
|
||||
run wget "$nps_baseurl/$nps_url_fname.zip" -O "$nps_downloaded"
|
||||
nps_module_dir="$BUILDDIR/$nps_downloaded_fname"
|
||||
delete_if_already_exists "$nps_module_dir"
|
||||
echo "Extracting ngx_pagespeed..."
|
||||
run unzip -q "$nps_downloaded" -d "$BUILDDIR"
|
||||
run cd "$nps_module_dir"
|
||||
|
||||
# Now we need to figure out what precompiled version of PSOL to build
|
||||
# ngx_pagespeed against.
|
||||
if "$DRYRUN"; then
|
||||
psol_url="https://psol.example.com/cant-get-psol-version-in-dry-run.tar.gz"
|
||||
elif [ -e PSOL_BINARY_URL ]; then
|
||||
# Releases after 1.11.33.4 there is a PSOL_BINARY_URL file that tells us
|
||||
# where to look.
|
||||
psol_url="$(scripts/format_binary_url.sh PSOL_BINARY_URL)"
|
||||
if [[ "$psol_url" != https://* ]]; then
|
||||
fail "Got bad psol binary location information: $psol_url"
|
||||
fi
|
||||
else
|
||||
# For past releases we have to grep it from the config file. The url has
|
||||
# always looked like this, and the config file has contained it since before
|
||||
# we started tagging our ngx_pagespeed releases.
|
||||
psol_url="$(
|
||||
grep -o "https://dl.google.com/dl/page-speed/psol/[0-9.]*.tar.gz" config)"
|
||||
if [ -z "$psol_url" ]; then
|
||||
fail "Couldn't find PSOL url in $PWD/config"
|
||||
fi
|
||||
fi
|
||||
|
||||
run wget "$psol_url"
|
||||
echo "Extracting PSOL..."
|
||||
run tar -xzf $(basename "$psol_url") # extracts to psol/
|
||||
|
||||
if "$DYNAMIC_MODULE"; then
|
||||
add_module="--add-dynamic-module=$nps_module_dir"
|
||||
else
|
||||
add_module="--add-module=$nps_module_dir"
|
||||
fi
|
||||
configure_args=("$add_module" "${extra_flags[@]}")
|
||||
|
||||
echo
|
||||
if [ -z "$NGINX_VERSION" ]; then
|
||||
# They didn't specify an nginx version, so we're just preparing the
|
||||
# module for them to install.
|
||||
echo "ngx_pagespeed is ready to be built against nginx."
|
||||
echo "When running ./configure pass in:"
|
||||
echo " $(quote_arguments "${configure_args[@]}")"
|
||||
if [ ${#extra_flags[@]} -eq 0 ]; then
|
||||
echo "If this is for integration with an already-built nginx, make sure"
|
||||
echo "to include any other arguments you originally passed to ./configure"
|
||||
echo "You can see these with 'nginx -V'."
|
||||
else
|
||||
echo "Note: because we need to set $(quote_arguments "${extra_flags[@]}")"
|
||||
echo "on this platform, if you want to integrate ngx_pagespeed with an"
|
||||
echo "already-built nginx you're going to need to rebuild your nginx with"
|
||||
echo "those flags set."
|
||||
fi
|
||||
else
|
||||
# Download and build nginx.
|
||||
nginx_leaf="nginx-${NGINX_VERSION}.tar.gz"
|
||||
nginx_fname="$TEMPDIR/$nginx_leaf"
|
||||
run wget "http://nginx.org/download/$nginx_leaf" -O "$nginx_fname"
|
||||
nginx_dir="$BUILDDIR/nginx-${NGINX_VERSION}/"
|
||||
delete_if_already_exists "$nginx_dir"
|
||||
echo "Extracting nginx..."
|
||||
run tar -xzf "$nginx_fname" --directory "$BUILDDIR"
|
||||
"$DRYRUN" || cd "$nginx_dir"
|
||||
|
||||
configure=("./configure" "${configure_args[@]}")
|
||||
echo "About to build nginx. Do you have any additional ./configure"
|
||||
echo "arguments you would like to set? For example, if you would like"
|
||||
echo "to build nginx with https support give --with-http_ssl_module"
|
||||
echo "If you don't have any, just press enter."
|
||||
read -p "> " additional_configure_args
|
||||
if [ -n "$additional_configure_args" ]; then
|
||||
# Split additional_configure_args respecting any internal quotation.
|
||||
# Otherwise things like --with-cc-opt='-foo -bar' won't work.
|
||||
eval additional_configure_args=("$additional_configure_args")
|
||||
configure=("${configure[@]}" "${additional_configure_args[@]}")
|
||||
fi
|
||||
echo "About to configure nginx with:"
|
||||
echo " $(quote_arguments "${configure[@]}")"
|
||||
continue_or_exit "Does this look right?"
|
||||
run "${configure[@]}"
|
||||
|
||||
continue_or_exit "Build nginx?"
|
||||
run make
|
||||
|
||||
continue_or_exit "Install nginx?"
|
||||
run sudo make install
|
||||
|
||||
echo
|
||||
if "$DYNAMIC_MODULE"; then
|
||||
echo "Nginx installed with ngx_pagespeed support available as a"
|
||||
echo "loadable module."
|
||||
echo
|
||||
echo "To load the ngx_pagespeed module, you'll need to add:"
|
||||
echo " load_module \"modules/ngx_pagespeed.so\";"
|
||||
echo "at the top of your main nginx configuration file."
|
||||
else
|
||||
echo "Nginx installed with ngx_pagespeed support compiled-in."
|
||||
fi
|
||||
echo
|
||||
echo "If this is a new installation you probably need an init script to"
|
||||
echo "manage starting and stopping the nginx service. See:"
|
||||
echo " http://wiki.nginx.org/InitScripts"
|
||||
echo
|
||||
echo "You'll also need to configure ngx_pagespeed if you haven't yet:"
|
||||
echo " https://developers.google.com/speed/pagespeed/module/configuration"
|
||||
fi
|
||||
if "$DRYRUN"; then
|
||||
echo "[this was a dry run; your system is unchanged]"
|
||||
fi
|
||||
}
|
||||
|
||||
# Start running things from a call at the end so if this script is executed
|
||||
# after a partial download it doesn't do anything.
|
||||
build_ngx_pagespeed "$@"
|
||||
24
ngx_pagespeed-latest-beta/scripts/format_binary_url.sh
Executable file
24
ngx_pagespeed-latest-beta/scripts/format_binary_url.sh
Executable file
@@ -0,0 +1,24 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
set -u
|
||||
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "Usage: $(basename $0) <url_file>" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
url_file=$1
|
||||
|
||||
if [ ! -e "$url_file" ]; then
|
||||
echo "Url file '$url_file' missing!" >&2
|
||||
fi
|
||||
|
||||
# The size names must match install/build_psol.sh in mod_pagespeed
|
||||
if [ "$(uname -m)" = x86_64 ]; then
|
||||
bit_size_name=x64
|
||||
else
|
||||
bit_size_name=ia32
|
||||
fi
|
||||
|
||||
sed -e 's/$BIT_SIZE_NAME\b/'$bit_size_name'/g' $url_file
|
||||
22
ngx_pagespeed-latest-beta/scripts/pagespeed_libraries_generator.sh
Executable file
22
ngx_pagespeed-latest-beta/scripts/pagespeed_libraries_generator.sh
Executable file
@@ -0,0 +1,22 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Converts pagespeed_libraries.conf from Apache-format to Nginx-format,
|
||||
# supporting the canonicalize_javascript_libraries filter.
|
||||
# Inspired by https://github.com/pagespeed/ngx_pagespeed/issues/532
|
||||
#
|
||||
# Usage:
|
||||
# scripts/pagespeed_libraries_generator.sh > pagespeed_libraries.conf
|
||||
#
|
||||
# Then have nginx include that configuration file and enable the filter
|
||||
# canonicalize_javascript_libraries.
|
||||
#
|
||||
# Author: vid@zippykid.com (Vid Luther)
|
||||
# jefftk@google.com (Jeff Kaufman)
|
||||
|
||||
URL="https://github.com/pagespeed/mod_pagespeed/raw/master/"
|
||||
URL+="net/instaweb/genfiles/conf/pagespeed_libraries.conf"
|
||||
curl -L -s -S "$URL" \
|
||||
| grep ModPagespeedLibrary \
|
||||
| while read library size hash url ; do
|
||||
echo " pagespeed Library $size $hash $url;"
|
||||
done
|
||||
114
ngx_pagespeed-latest-beta/src/log_message_handler.cc
Normal file
114
ngx_pagespeed-latest-beta/src/log_message_handler.cc
Normal file
@@ -0,0 +1,114 @@
|
||||
// Copyright 2010 Google Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// Author: jmarantz@google.com (Joshua Marantz)
|
||||
// Author: sligocki@google.com (Shawn Ligocki)
|
||||
// Author: jefftk@google.com (Jeff Kaufman)
|
||||
|
||||
// TODO(jefftk): share more of this code with apache's log_message_handler
|
||||
|
||||
#include "log_message_handler.h"
|
||||
|
||||
#include <unistd.h>
|
||||
|
||||
#include <limits>
|
||||
#include <string>
|
||||
|
||||
#include "base/debug/debugger.h"
|
||||
#include "base/debug/stack_trace.h"
|
||||
#include "base/logging.h"
|
||||
#include "net/instaweb/public/version.h"
|
||||
#include "pagespeed/kernel/base/string_util.h"
|
||||
|
||||
// Make sure we don't attempt to use LOG macros here, since doing so
|
||||
// would cause us to go into an infinite log loop.
|
||||
#undef LOG
|
||||
#define LOG USING_LOG_HERE_WOULD_CAUSE_INFINITE_RECURSION
|
||||
|
||||
namespace {
|
||||
|
||||
ngx_log_t* ngx_log = NULL;
|
||||
|
||||
ngx_uint_t GetNgxLogLevel(int severity) {
|
||||
switch (severity) {
|
||||
case logging::LOG_INFO:
|
||||
return NGX_LOG_INFO;
|
||||
case logging::LOG_WARNING:
|
||||
return NGX_LOG_WARN;
|
||||
case logging::LOG_ERROR:
|
||||
return NGX_LOG_ERR;
|
||||
case logging::LOG_ERROR_REPORT:
|
||||
case logging::LOG_FATAL:
|
||||
return NGX_LOG_ALERT;
|
||||
default: // For VLOG(s)
|
||||
return NGX_LOG_DEBUG;
|
||||
}
|
||||
}
|
||||
|
||||
bool LogMessageHandler(int severity, const char* file, int line,
|
||||
size_t message_start, const GoogleString& str) {
|
||||
ngx_uint_t this_log_level = GetNgxLogLevel(severity);
|
||||
|
||||
GoogleString message = str;
|
||||
if (severity == logging::LOG_FATAL) {
|
||||
if (base::debug::BeingDebugged()) {
|
||||
base::debug::BreakDebugger();
|
||||
} else {
|
||||
base::debug::StackTrace trace;
|
||||
std::ostringstream stream;
|
||||
trace.OutputToStream(&stream);
|
||||
message.append(stream.str());
|
||||
}
|
||||
}
|
||||
|
||||
// Trim the newline off the end of the message string.
|
||||
size_t last_msg_character_index = message.length() - 1;
|
||||
if (message[last_msg_character_index] == '\n') {
|
||||
message.resize(last_msg_character_index);
|
||||
}
|
||||
|
||||
ngx_log_error(this_log_level, ngx_log, 0, "[ngx_pagespeed %s] %s",
|
||||
net_instaweb::kModPagespeedVersion,
|
||||
message.c_str());
|
||||
|
||||
if (severity == logging::LOG_FATAL) {
|
||||
// Crash the process to generate a dump.
|
||||
base::debug::BreakDebugger();
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
|
||||
namespace net_instaweb {
|
||||
|
||||
namespace log_message_handler {
|
||||
|
||||
|
||||
void Install(ngx_log_t* log_in) {
|
||||
ngx_log = log_in;
|
||||
logging::SetLogMessageHandler(&LogMessageHandler);
|
||||
|
||||
// All VLOG(2) and higher will be displayed as DEBUG logs if the nginx log
|
||||
// level is DEBUG.
|
||||
if (ngx_log->log_level >= NGX_LOG_DEBUG) {
|
||||
logging::SetMinLogLevel(-2);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace log_message_handler
|
||||
|
||||
} // namespace net_instaweb
|
||||
44
ngx_pagespeed-latest-beta/src/log_message_handler.h
Normal file
44
ngx_pagespeed-latest-beta/src/log_message_handler.h
Normal file
@@ -0,0 +1,44 @@
|
||||
// Copyright 2010 Google Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// Author: jmarantz@google.com (Joshua Marantz)
|
||||
// Author: sligocki@google.com (Shawn Ligocki)
|
||||
// Author: jefftk@google.com (Jeff Kaufman)
|
||||
|
||||
#ifndef LOG_MESSAGE_HANDLER_H_
|
||||
#define LOG_MESSAGE_HANDLER_H_
|
||||
|
||||
extern "C" {
|
||||
#include <ngx_config.h>
|
||||
#include <ngx_core.h>
|
||||
#include <ngx_http.h>
|
||||
#include <ngx_log.h>
|
||||
}
|
||||
|
||||
namespace net_instaweb {
|
||||
|
||||
namespace log_message_handler {
|
||||
|
||||
// Install a log message handler that routes LOG() messages to the
|
||||
// server error log. Should be called once at startup. If server blocks define
|
||||
// their own logging files you would expect that LOG() messages would be routed
|
||||
// appropriately, but because logging is all done with global variables this
|
||||
// isn't possible.
|
||||
void Install(ngx_log_t* log_in);
|
||||
|
||||
} // namespace log_message_handler
|
||||
|
||||
} // namespace net_instaweb
|
||||
|
||||
#endif // LOG_MESSAGE_HANDLER_H_
|
||||
360
ngx_pagespeed-latest-beta/src/ngx_base_fetch.cc
Normal file
360
ngx_pagespeed-latest-beta/src/ngx_base_fetch.cc
Normal file
@@ -0,0 +1,360 @@
|
||||
/*
|
||||
* Copyright 2012 Google Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
* Author: jefftk@google.com (Jeff Kaufman)
|
||||
*/
|
||||
|
||||
#include "ngx_pagespeed.h" // Must come first, see comments in CollectHeaders.
|
||||
|
||||
#include <unistd.h> //for usleep
|
||||
|
||||
#include "ngx_base_fetch.h"
|
||||
#include "ngx_event_connection.h"
|
||||
#include "ngx_list_iterator.h"
|
||||
|
||||
#include "net/instaweb/rewriter/public/rewrite_driver.h"
|
||||
#include "net/instaweb/rewriter/public/rewrite_options.h"
|
||||
#include "net/instaweb/rewriter/public/rewrite_stats.h"
|
||||
#include "pagespeed/kernel/base/google_message_handler.h"
|
||||
#include "pagespeed/kernel/base/message_handler.h"
|
||||
#include "pagespeed/kernel/base/posix_timer.h"
|
||||
#include "pagespeed/kernel/http/response_headers.h"
|
||||
|
||||
namespace net_instaweb {
|
||||
|
||||
const char kHeadersComplete = 'H';
|
||||
const char kFlush = 'F';
|
||||
const char kDone = 'D';
|
||||
|
||||
NgxEventConnection* NgxBaseFetch::event_connection = NULL;
|
||||
int NgxBaseFetch::active_base_fetches = 0;
|
||||
|
||||
NgxBaseFetch::NgxBaseFetch(StringPiece url,
|
||||
ngx_http_request_t* r,
|
||||
NgxServerContext* server_context,
|
||||
const RequestContextPtr& request_ctx,
|
||||
PreserveCachingHeaders preserve_caching_headers,
|
||||
NgxBaseFetchType base_fetch_type,
|
||||
const RewriteOptions* options)
|
||||
: AsyncFetch(request_ctx),
|
||||
url_(url.data(), url.size()),
|
||||
request_(r),
|
||||
server_context_(server_context),
|
||||
options_(options),
|
||||
need_flush_(false),
|
||||
done_called_(false),
|
||||
last_buf_sent_(false),
|
||||
references_(2),
|
||||
base_fetch_type_(base_fetch_type),
|
||||
preserve_caching_headers_(preserve_caching_headers),
|
||||
detached_(false),
|
||||
suppress_(false) {
|
||||
if (pthread_mutex_init(&mutex_, NULL)) CHECK(0);
|
||||
__sync_add_and_fetch(&NgxBaseFetch::active_base_fetches, 1);
|
||||
}
|
||||
|
||||
NgxBaseFetch::~NgxBaseFetch() {
|
||||
pthread_mutex_destroy(&mutex_);
|
||||
__sync_add_and_fetch(&NgxBaseFetch::active_base_fetches, -1);
|
||||
}
|
||||
|
||||
bool NgxBaseFetch::Initialize(ngx_cycle_t* cycle) {
|
||||
CHECK(event_connection == NULL) << "event connection already set";
|
||||
event_connection = new NgxEventConnection(ReadCallback);
|
||||
return event_connection->Init(cycle);
|
||||
}
|
||||
|
||||
void NgxBaseFetch::Terminate() {
|
||||
if (event_connection != NULL) {
|
||||
GoogleMessageHandler handler;
|
||||
PosixTimer timer;
|
||||
int64 timeout_us = Timer::kSecondUs * 30;
|
||||
int64 end_us = timer.NowUs() + timeout_us;
|
||||
static unsigned int sleep_microseconds = 100;
|
||||
|
||||
handler.Message(
|
||||
kInfo,"NgxBaseFetch::Terminate rounding up %d active base fetches.",
|
||||
NgxBaseFetch::active_base_fetches);
|
||||
|
||||
// Try to continue processing and get the active base fetch count to 0
|
||||
// untill the timeout expires.
|
||||
// TODO(oschaaf): This needs more work.
|
||||
while (NgxBaseFetch::active_base_fetches > 0 && end_us > timer.NowUs()) {
|
||||
event_connection->Drain();
|
||||
usleep(sleep_microseconds);
|
||||
}
|
||||
|
||||
if (NgxBaseFetch::active_base_fetches != 0) {
|
||||
handler.Message(
|
||||
kWarning,"NgxBaseFetch::Terminate timed out with %d active base fetches.",
|
||||
NgxBaseFetch::active_base_fetches);
|
||||
}
|
||||
|
||||
// Close down the named pipe.
|
||||
event_connection->Shutdown();
|
||||
delete event_connection;
|
||||
event_connection = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
const char* BaseFetchTypeToCStr(NgxBaseFetchType type) {
|
||||
switch(type) {
|
||||
case kPageSpeedResource:
|
||||
return "ps resource";
|
||||
case kHtmlTransform:
|
||||
return "html transform";
|
||||
case kAdminPage:
|
||||
return "admin page";
|
||||
case kIproLookup:
|
||||
return "ipro lookup";
|
||||
case kPageSpeedProxy:
|
||||
return "pagespeed proxy";
|
||||
}
|
||||
CHECK(false);
|
||||
return "can't get here";
|
||||
}
|
||||
|
||||
// TODO(oschaaf): replace the ngx_log_error with VLOGS or pass in a
|
||||
// MessageHandler and use that.
|
||||
void NgxBaseFetch::ReadCallback(const ps_event_data& data) {
|
||||
NgxBaseFetch* base_fetch = reinterpret_cast<NgxBaseFetch*>(data.sender);
|
||||
ngx_http_request_t* r = base_fetch->request();
|
||||
bool detached = base_fetch->detached();
|
||||
#if (NGX_DEBUG) // `type` is unused if NGX_DEBUG isn't set, needed for -Werror.
|
||||
const char* type = BaseFetchTypeToCStr(base_fetch->base_fetch_type_);
|
||||
#endif
|
||||
int refcount = base_fetch->DecrementRefCount();
|
||||
|
||||
#if (NGX_DEBUG)
|
||||
ngx_log_error(NGX_LOG_DEBUG, ngx_cycle->log, 0,
|
||||
"pagespeed [%p] event: %c. bf:%p (%s) - refcnt:%d - det: %c", r,
|
||||
data.type, base_fetch, type, refcount, detached ? 'Y': 'N');
|
||||
#endif
|
||||
|
||||
// If we ended up destructing the base fetch, or the request context is
|
||||
// detached, skip this event.
|
||||
if (refcount == 0 || detached) {
|
||||
return;
|
||||
}
|
||||
|
||||
ps_request_ctx_t* ctx = ps_get_request_context(r);
|
||||
|
||||
// If our request context was zeroed, skip this event.
|
||||
// See https://github.com/pagespeed/ngx_pagespeed/issues/1081
|
||||
if (ctx == NULL) {
|
||||
// Should not happen normally, when it does this message will cause our
|
||||
// system tests to fail.
|
||||
ngx_log_error(NGX_LOG_WARN, ngx_cycle->log, 0,
|
||||
"pagespeed [%p] skipping event: request context gone", r);
|
||||
return;
|
||||
}
|
||||
|
||||
// Normally we expect the sender to equal the active NgxBaseFetch instance.
|
||||
DCHECK(data.sender == ctx->base_fetch);
|
||||
|
||||
// If someone changed our request context or NgxBaseFetch, skip processing.
|
||||
if (data.sender != ctx->base_fetch) {
|
||||
ngx_log_error(NGX_LOG_WARN, ngx_cycle->log, 0,
|
||||
"pagespeed [%p] skipping event: event originating from disassociated"
|
||||
" NgxBaseFetch instance.", r);
|
||||
return;
|
||||
}
|
||||
|
||||
int rc;
|
||||
bool run_posted = true;
|
||||
// If we are unlucky enough to have our connection finalized mid-ipro-lookup,
|
||||
// we must enter a different flow. Also see ps_in_place_check_header_filter().
|
||||
if ((ctx->base_fetch->base_fetch_type_ != kIproLookup)
|
||||
&& r->connection->error) {
|
||||
ngx_log_error(NGX_LOG_DEBUG, ngx_cycle->log, 0,
|
||||
"pagespeed [%p] request already finalized %d", r, r->count);
|
||||
rc = NGX_ERROR;
|
||||
run_posted = false;
|
||||
} else {
|
||||
rc = ps_base_fetch::ps_base_fetch_handler(r);
|
||||
}
|
||||
|
||||
#if (NGX_DEBUG)
|
||||
ngx_log_error(NGX_LOG_DEBUG, ngx_cycle->log, 0,
|
||||
"pagespeed [%p] ps_base_fetch_handler() returned %d for %c",
|
||||
r, rc, data.type);
|
||||
#endif
|
||||
|
||||
ngx_connection_t* c = r->connection;
|
||||
ngx_http_finalize_request(r, rc);
|
||||
|
||||
if (run_posted) {
|
||||
// See http://forum.nginx.org/read.php?2,253006,253061
|
||||
ngx_http_run_posted_requests(c);
|
||||
}
|
||||
}
|
||||
|
||||
void NgxBaseFetch::Lock() {
|
||||
pthread_mutex_lock(&mutex_);
|
||||
}
|
||||
|
||||
void NgxBaseFetch::Unlock() {
|
||||
pthread_mutex_unlock(&mutex_);
|
||||
}
|
||||
|
||||
bool NgxBaseFetch::HandleWrite(const StringPiece& sp,
|
||||
MessageHandler* handler) {
|
||||
Lock();
|
||||
buffer_.append(sp.data(), sp.size());
|
||||
Unlock();
|
||||
return true;
|
||||
}
|
||||
|
||||
// should only be called in nginx thread
|
||||
ngx_int_t NgxBaseFetch::CopyBufferToNginx(ngx_chain_t** link_ptr) {
|
||||
CHECK(!(done_called_ && last_buf_sent_))
|
||||
<< "CopyBufferToNginx() was called after the last buffer was sent";
|
||||
|
||||
// there is no buffer to send
|
||||
if (!done_called_ && buffer_.empty()) {
|
||||
*link_ptr = NULL;
|
||||
return NGX_AGAIN;
|
||||
}
|
||||
|
||||
int rc = string_piece_to_buffer_chain(request_->pool, buffer_, link_ptr,
|
||||
done_called_ /* send_last_buf */,
|
||||
need_flush_);
|
||||
need_flush_ = false;
|
||||
|
||||
if (rc != NGX_OK) {
|
||||
return rc;
|
||||
}
|
||||
|
||||
// Done with buffer contents now.
|
||||
buffer_.clear();
|
||||
|
||||
if (done_called_) {
|
||||
last_buf_sent_ = true;
|
||||
return NGX_OK;
|
||||
}
|
||||
|
||||
return NGX_AGAIN;
|
||||
}
|
||||
|
||||
// There may also be a race condition if this is called between the last Write()
|
||||
// and Done() such that we're sending an empty buffer with last_buf set, which I
|
||||
// think nginx will reject.
|
||||
ngx_int_t NgxBaseFetch::CollectAccumulatedWrites(ngx_chain_t** link_ptr) {
|
||||
ngx_int_t rc;
|
||||
Lock();
|
||||
rc = CopyBufferToNginx(link_ptr);
|
||||
Unlock();
|
||||
return rc;
|
||||
}
|
||||
|
||||
ngx_int_t NgxBaseFetch::CollectHeaders(ngx_http_headers_out_t* headers_out) {
|
||||
// nginx defines _FILE_OFFSET_BITS to 64, which changes the size of off_t.
|
||||
// If a standard header is accidentally included before the nginx header,
|
||||
// on a 32-bit system off_t will be 4 bytes and we don't assign all the
|
||||
// bits of content_length_n. Sanity check that did not happen.
|
||||
// This could use static_assert, but this file is not built with --std=c++11.
|
||||
bool sanity_check_off_t[sizeof(off_t) == 8 ? 1 : -1] __attribute__ ((unused));
|
||||
|
||||
const ResponseHeaders* pagespeed_headers = response_headers();
|
||||
|
||||
if (content_length_known()) {
|
||||
headers_out->content_length = NULL;
|
||||
headers_out->content_length_n = content_length();
|
||||
}
|
||||
|
||||
return copy_response_headers_to_ngx(request_, *pagespeed_headers,
|
||||
preserve_caching_headers_);
|
||||
}
|
||||
|
||||
void NgxBaseFetch::RequestCollection(char type) {
|
||||
if (suppress_) {
|
||||
return;
|
||||
}
|
||||
|
||||
// We must optimistically increment the refcount, and decrement it
|
||||
// when we conclude we failed. If we only increment on a successfull write,
|
||||
// there's a small chance that between writing and adding to the refcount
|
||||
// both pagespeed and nginx will release their refcount -- destructing
|
||||
// this NgxBaseFetch instance.
|
||||
IncrementRefCount();
|
||||
if (!event_connection->WriteEvent(type, this)) {
|
||||
DecrementRefCount();
|
||||
}
|
||||
}
|
||||
|
||||
void NgxBaseFetch::HandleHeadersComplete() {
|
||||
int status_code = response_headers()->status_code();
|
||||
bool status_ok = (status_code != 0) && (status_code < 400);
|
||||
|
||||
if ((base_fetch_type_ != kIproLookup) || status_ok) {
|
||||
// If this is a 404 response we need to count it in the stats.
|
||||
if (response_headers()->status_code() == HttpStatus::kNotFound) {
|
||||
server_context_->rewrite_stats()->resource_404_count()->Add(1);
|
||||
}
|
||||
}
|
||||
|
||||
RequestCollection(kHeadersComplete); // Headers available.
|
||||
|
||||
// For the IPRO lookup, supress notification of the nginx side here.
|
||||
// If we send both the headerscomplete event and the one from done, nasty
|
||||
// stuff will happen if we loose the race with with the nginx side destructing
|
||||
// this base fetch instance.
|
||||
if (base_fetch_type_ == kIproLookup && !status_ok) {
|
||||
suppress_ = true;
|
||||
}
|
||||
}
|
||||
|
||||
bool NgxBaseFetch::HandleFlush(MessageHandler* handler) {
|
||||
Lock();
|
||||
need_flush_ = true;
|
||||
Unlock();
|
||||
RequestCollection(kFlush); // A new part of the response body is available
|
||||
return true;
|
||||
}
|
||||
|
||||
int NgxBaseFetch::DecrementRefCount() {
|
||||
return DecrefAndDeleteIfUnreferenced();
|
||||
}
|
||||
|
||||
int NgxBaseFetch::IncrementRefCount() {
|
||||
return __sync_add_and_fetch(&references_, 1);
|
||||
}
|
||||
|
||||
int NgxBaseFetch::DecrefAndDeleteIfUnreferenced() {
|
||||
// Creates a full memory barrier.
|
||||
int r = __sync_add_and_fetch(&references_, -1);
|
||||
if (r == 0) {
|
||||
delete this;
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
void NgxBaseFetch::HandleDone(bool success) {
|
||||
// TODO(jefftk): it's possible that instead of locking here we can just modify
|
||||
// CopyBufferToNginx to only read done_called_ once.
|
||||
CHECK(!done_called_) << "Done already called!";
|
||||
Lock();
|
||||
done_called_ = true;
|
||||
Unlock();
|
||||
RequestCollection(kDone);
|
||||
DecrefAndDeleteIfUnreferenced();
|
||||
}
|
||||
|
||||
bool NgxBaseFetch::IsCachedResultValid(const ResponseHeaders& headers) {
|
||||
return OptionsAwareHTTPCacheCallback::IsCacheValid(
|
||||
url_, *options_, request_context(), headers);
|
||||
}
|
||||
|
||||
} // namespace net_instaweb
|
||||
192
ngx_pagespeed-latest-beta/src/ngx_base_fetch.h
Normal file
192
ngx_pagespeed-latest-beta/src/ngx_base_fetch.h
Normal file
@@ -0,0 +1,192 @@
|
||||
/*
|
||||
* Copyright 2012 Google Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Author: jefftk@google.com (Jeff Kaufman)
|
||||
//
|
||||
// Collects output from pagespeed and buffers it until nginx asks for it.
|
||||
// Notifies nginx via NgxEventConnection to call ReadCallback() when
|
||||
// the headers are computed, when a flush should be performed, and when done.
|
||||
//
|
||||
// - nginx creates a base fetch and passes it to a new proxy fetch.
|
||||
// - The proxy fetch manages rewriting and thread complexity, and through
|
||||
// several chained steps passes rewritten html to HandleWrite().
|
||||
// - Written data is buffered.
|
||||
// - When HandleHeadersComplete(), HandleFlush(), or HandleDone() is called by
|
||||
// PSOL, events are written to NgxEventConnection which will end up being
|
||||
// handled by ReadCallback() on nginx's thread.
|
||||
// When applicable, request processing will be continued via a call to
|
||||
// ps_base_fetch_handler().
|
||||
// - ps_base_fetch_handler() will pull the header and body bytes from PSOL
|
||||
// via CollectAccumulatedWrites() and write those to the module's output.
|
||||
//
|
||||
// This class is referred to in three places: the proxy fetch, nginx's request,
|
||||
// and pending events written to the associated NgxEventConnection. It must stay
|
||||
// alive until the proxy fetch and nginx request are finished, and no more
|
||||
// events are pending.
|
||||
// - The proxy fetch will call Done() to indicate this.
|
||||
// - nginx will call Detach() when the associated request is handled
|
||||
// completely (e.g. the request context is about to be destroyed).
|
||||
// - ReadCallback() will call DecrementRefCount() on instances associated to
|
||||
// events it handles.
|
||||
//
|
||||
// When the last reference is dropped, this class will delete itself.
|
||||
//
|
||||
// TODO(jmarantz): consider sharing the cache-invalidation infrastructure
|
||||
// with ApacheFetch, using a common base class.
|
||||
|
||||
#ifndef NGX_BASE_FETCH_H_
|
||||
#define NGX_BASE_FETCH_H_
|
||||
|
||||
extern "C" {
|
||||
#include <ngx_http.h>
|
||||
}
|
||||
|
||||
#include <pthread.h>
|
||||
|
||||
#include "ngx_pagespeed.h"
|
||||
|
||||
#include "ngx_event_connection.h"
|
||||
#include "ngx_server_context.h"
|
||||
|
||||
#include "net/instaweb/http/public/async_fetch.h"
|
||||
#include "net/instaweb/rewriter/public/rewrite_options.h"
|
||||
#include "pagespeed/kernel/base/string.h"
|
||||
#include "pagespeed/kernel/http/headers.h"
|
||||
|
||||
namespace net_instaweb {
|
||||
|
||||
enum NgxBaseFetchType {
|
||||
kIproLookup,
|
||||
kHtmlTransform,
|
||||
kPageSpeedResource,
|
||||
kAdminPage,
|
||||
kPageSpeedProxy
|
||||
};
|
||||
|
||||
class NgxBaseFetch : public AsyncFetch {
|
||||
public:
|
||||
NgxBaseFetch(StringPiece url, ngx_http_request_t* r,
|
||||
NgxServerContext* server_context,
|
||||
const RequestContextPtr& request_ctx,
|
||||
PreserveCachingHeaders preserve_caching_headers,
|
||||
NgxBaseFetchType base_fetch_type,
|
||||
const RewriteOptions* options);
|
||||
virtual ~NgxBaseFetch();
|
||||
|
||||
// Statically initializes event_connection, require for PSOL and nginx to
|
||||
// communicate.
|
||||
static bool Initialize(ngx_cycle_t* cycle);
|
||||
|
||||
// Attempts to finish up request processing queued up in the named pipe and
|
||||
// PSOL for a fixed amount of time. If time is up, a fast and rough shutdown
|
||||
// is attempted.
|
||||
// Statically terminates and NULLS event_connection.
|
||||
static void Terminate();
|
||||
|
||||
static void ReadCallback(const ps_event_data& data);
|
||||
|
||||
// Puts a chain in link_ptr if we have any output data buffered. Returns
|
||||
// NGX_OK on success, NGX_ERROR on errors. If there's no data to send, sends
|
||||
// data only if Done() has been called. Indicates the end of output by
|
||||
// setting last_buf on the last buffer in the chain.
|
||||
//
|
||||
// Sets link_ptr to a chain of as many buffers are needed for the output.
|
||||
//
|
||||
// Called by nginx in response to seeing a byte on the pipe.
|
||||
ngx_int_t CollectAccumulatedWrites(ngx_chain_t** link_ptr);
|
||||
|
||||
// Copies response headers into headers_out.
|
||||
//
|
||||
// Called by nginx before calling CollectAccumulatedWrites() for the first
|
||||
// time for resource fetches. Not called at all for proxy fetches.
|
||||
ngx_int_t CollectHeaders(ngx_http_headers_out_t* headers_out);
|
||||
|
||||
// Called by nginx to decrement the refcount.
|
||||
int DecrementRefCount();
|
||||
|
||||
// Called by pagespeed to increment the refcount.
|
||||
int IncrementRefCount();
|
||||
|
||||
// Detach() is called when the nginx side releases this base fetch. It
|
||||
// sets detached_ to true and decrements the refcount. We need to know
|
||||
// this to be able to handle events which nginx request context has been
|
||||
// released while the event was in-flight.
|
||||
void Detach() { detached_ = true; DecrementRefCount(); }
|
||||
|
||||
bool detached() { return detached_; }
|
||||
|
||||
ngx_http_request_t* request() { return request_; }
|
||||
NgxBaseFetchType base_fetch_type() { return base_fetch_type_; }
|
||||
|
||||
bool IsCachedResultValid(const ResponseHeaders& headers) override;
|
||||
|
||||
private:
|
||||
virtual bool HandleWrite(const StringPiece& sp, MessageHandler* handler);
|
||||
virtual bool HandleFlush(MessageHandler* handler);
|
||||
virtual void HandleHeadersComplete();
|
||||
virtual void HandleDone(bool success);
|
||||
|
||||
// Indicate to nginx that we would like it to call
|
||||
// CollectAccumulatedWrites().
|
||||
void RequestCollection(char type);
|
||||
|
||||
// Lock must be acquired first.
|
||||
// Returns:
|
||||
// NGX_ERROR: failure
|
||||
// NGX_AGAIN: still has buffer to send, need to checkout link_ptr
|
||||
// NGX_OK: done, HandleDone has been called
|
||||
// Allocates an nginx buffer, copies our buffer_ contents into it, clears
|
||||
// buffer_.
|
||||
ngx_int_t CopyBufferToNginx(ngx_chain_t** link_ptr);
|
||||
|
||||
void Lock();
|
||||
void Unlock();
|
||||
|
||||
// Called by Done() and Release(). Decrements our reference count, and if
|
||||
// it's zero we delete ourself.
|
||||
int DecrefAndDeleteIfUnreferenced();
|
||||
|
||||
static NgxEventConnection* event_connection;
|
||||
|
||||
// Live count of NgxBaseFetch instances that are currently in use.
|
||||
static int active_base_fetches;
|
||||
|
||||
GoogleString url_;
|
||||
ngx_http_request_t* request_;
|
||||
GoogleString buffer_;
|
||||
NgxServerContext* server_context_;
|
||||
const RewriteOptions* options_;
|
||||
bool need_flush_;
|
||||
bool done_called_;
|
||||
bool last_buf_sent_;
|
||||
// How many active references there are to this fetch. Starts at two,
|
||||
// decremented once when Done() is called and once when Detach() is called.
|
||||
// Incremented for each event written by pagespeed for this NgxBaseFetch, and
|
||||
// decremented on the nginx side for each event read for it.
|
||||
int references_;
|
||||
pthread_mutex_t mutex_;
|
||||
NgxBaseFetchType base_fetch_type_;
|
||||
PreserveCachingHeaders preserve_caching_headers_;
|
||||
// Set to true just before the nginx side releases its reference
|
||||
bool detached_;
|
||||
bool suppress_;
|
||||
|
||||
DISALLOW_COPY_AND_ASSIGN(NgxBaseFetch);
|
||||
};
|
||||
|
||||
} // namespace net_instaweb
|
||||
|
||||
#endif // NGX_BASE_FETCH_H_
|
||||
51
ngx_pagespeed-latest-beta/src/ngx_caching_headers.cc
Normal file
51
ngx_pagespeed-latest-beta/src/ngx_caching_headers.cc
Normal file
@@ -0,0 +1,51 @@
|
||||
/*
|
||||
* Copyright 2013 Google Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Author: jefftk@google.com (Jeff Kaufman)
|
||||
|
||||
#include "ngx_caching_headers.h"
|
||||
#include "ngx_list_iterator.h"
|
||||
|
||||
#include "ngx_pagespeed.h"
|
||||
|
||||
namespace net_instaweb {
|
||||
|
||||
bool NgxCachingHeaders::Lookup(const StringPiece& key,
|
||||
StringPieceVector* values) {
|
||||
ngx_table_elt_t* header;
|
||||
NgxListIterator it(&(request_->headers_out.headers.part));
|
||||
while ((header = it.Next()) != NULL) {
|
||||
if (header->hash != 0 && key == str_to_string_piece(header->key)) {
|
||||
// This will be called multiple times if there are multiple headers with
|
||||
// this name. Each time it will append to values.
|
||||
SplitStringPieceToVector(str_to_string_piece(header->value), ",", values,
|
||||
true /* omit empty strings */);
|
||||
}
|
||||
}
|
||||
|
||||
if (values->size() == 0) {
|
||||
// No header found with this name.
|
||||
return false;
|
||||
}
|
||||
|
||||
for (int i = 0, n = values->size(); i < n; ++i) {
|
||||
TrimWhitespace(&((*values)[i]));
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace net_instaweb
|
||||
60
ngx_pagespeed-latest-beta/src/ngx_caching_headers.h
Normal file
60
ngx_pagespeed-latest-beta/src/ngx_caching_headers.h
Normal file
@@ -0,0 +1,60 @@
|
||||
/*
|
||||
* Copyright 2013 Google Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Author: jefftk@google.com (Jeff Kaufman)
|
||||
|
||||
#ifndef NGX_CACHING_HEADERS_H_
|
||||
#define NGX_CACHING_HEADERS_H_
|
||||
|
||||
extern "C" {
|
||||
#include <ngx_http.h>
|
||||
}
|
||||
|
||||
#include "pagespeed/kernel/base/basictypes.h"
|
||||
#include "pagespeed/kernel/http/caching_headers.h"
|
||||
|
||||
namespace net_instaweb {
|
||||
|
||||
// Based off of ApacheCachingHeaders in net/instaweb/apache/header_util.cc
|
||||
// Needed so ps_header_filter can call GenerateDisabledCacheControl().
|
||||
class NgxCachingHeaders : public CachingHeaders {
|
||||
public:
|
||||
explicit NgxCachingHeaders(ngx_http_request_t* request)
|
||||
: CachingHeaders(request->headers_out.status),
|
||||
request_(request) {
|
||||
}
|
||||
|
||||
virtual bool Lookup(const StringPiece& key, StringPieceVector* values);
|
||||
|
||||
virtual bool IsLikelyStaticResourceType() const {
|
||||
DCHECK(false); // not called in our use-case.
|
||||
return false;
|
||||
}
|
||||
|
||||
virtual bool IsCacheableResourceStatusCode() const {
|
||||
DCHECK(false); // not called in our use-case.
|
||||
return false;
|
||||
}
|
||||
|
||||
private:
|
||||
ngx_http_request_t* request_;
|
||||
|
||||
DISALLOW_COPY_AND_ASSIGN(NgxCachingHeaders);
|
||||
};
|
||||
|
||||
} // namespace net_instaweb
|
||||
|
||||
#endif // NGX_CACHING_HEADERS_H_
|
||||
172
ngx_pagespeed-latest-beta/src/ngx_event_connection.cc
Normal file
172
ngx_pagespeed-latest-beta/src/ngx_event_connection.cc
Normal file
@@ -0,0 +1,172 @@
|
||||
/*
|
||||
* Copyright 2014 Google Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Author: oschaaf@we-amp.com (Otto van der Schaaf)
|
||||
|
||||
extern "C" {
|
||||
|
||||
#include <ngx_channel.h>
|
||||
|
||||
}
|
||||
|
||||
#include "ngx_event_connection.h"
|
||||
|
||||
#include "pagespeed/kernel/base/google_message_handler.h"
|
||||
#include "pagespeed/kernel/base/message_handler.h"
|
||||
|
||||
namespace net_instaweb {
|
||||
|
||||
NgxEventConnection::NgxEventConnection(callbackPtr callback)
|
||||
: event_handler_(callback) {
|
||||
}
|
||||
|
||||
bool NgxEventConnection::Init(ngx_cycle_t* cycle) {
|
||||
int file_descriptors[2];
|
||||
|
||||
if (pipe(file_descriptors) != 0) {
|
||||
ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, "pagespeed: pipe() failed");
|
||||
return false;
|
||||
}
|
||||
if (ngx_nonblocking(file_descriptors[0]) == -1) {
|
||||
ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno,
|
||||
ngx_nonblocking_n "pagespeed: pipe[0] failed");
|
||||
} else if (ngx_nonblocking(file_descriptors[1]) == -1) {
|
||||
ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno,
|
||||
ngx_nonblocking_n "pagespeed: pipe[1] failed");
|
||||
} else if (!CreateNgxConnection(cycle, file_descriptors[0])) {
|
||||
ngx_log_error(NGX_LOG_EMERG, cycle->log, 0,
|
||||
"pagespeed: failed to create connection.");
|
||||
} else {
|
||||
pipe_read_fd_ = file_descriptors[0];
|
||||
pipe_write_fd_ = file_descriptors[1];
|
||||
return true;
|
||||
}
|
||||
close(file_descriptors[0]);
|
||||
close(file_descriptors[1]);
|
||||
return false;
|
||||
}
|
||||
|
||||
bool NgxEventConnection::CreateNgxConnection(ngx_cycle_t* cycle,
|
||||
ngx_fd_t pipe_fd) {
|
||||
// pipe_fd (the read side of the pipe will end up as c->fd on the
|
||||
// underlying ngx_connection_t that gets created here)
|
||||
ngx_int_t rc = ngx_add_channel_event(cycle, pipe_fd, NGX_READ_EVENT,
|
||||
&NgxEventConnection::ReadEventHandler);
|
||||
return rc == NGX_OK;
|
||||
}
|
||||
|
||||
void NgxEventConnection::ReadEventHandler(ngx_event_t* ev) {
|
||||
ngx_connection_t* c = static_cast<ngx_connection_t*>(ev->data);
|
||||
ngx_int_t result = ngx_handle_read_event(ev, 0);
|
||||
if (result != NGX_OK) {
|
||||
CHECK(false) << "pagespeed: ngx_handle_read_event error: " << result;
|
||||
}
|
||||
|
||||
if (ev->timedout) {
|
||||
ev->timedout = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
if (!NgxEventConnection::ReadAndNotify(c->fd)) {
|
||||
// This was copied from ngx_channel_handler(): for epoll, we need to call
|
||||
// ngx_del_conn(). Sadly, no documentation as to why.
|
||||
if (ngx_event_flags & NGX_USE_EPOLL_EVENT) {
|
||||
ngx_del_conn(c, 0);
|
||||
}
|
||||
ngx_close_connection(c);
|
||||
ngx_del_event(ev, NGX_READ_EVENT, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// Deserialize ps_event_data's from the pipe as they become available.
|
||||
// Subsequently do some bookkeeping, cleanup, and error checking to keep
|
||||
// the mess out of ps_base_fetch_handler.
|
||||
bool NgxEventConnection::ReadAndNotify(ngx_fd_t fd) {
|
||||
while (true) {
|
||||
// We read only one ps_event_data at a time for now:
|
||||
// We can end up recursing all the way and end up calling ourselves here.
|
||||
// If that happens in the middle of looping over multiple ps_event_data's we
|
||||
// have obtained with read(), the results from the next read() will make us
|
||||
// process events out of order. Which can give headaches.
|
||||
// Alternatively, we could maintain a queue to make sure we process in
|
||||
// sequence
|
||||
ps_event_data data;
|
||||
ngx_int_t size = read(fd, static_cast<void*>(&data), sizeof(data));
|
||||
|
||||
if (size == -1) {
|
||||
if (errno == EINTR) {
|
||||
continue;
|
||||
// TODO(oschaaf): should we worry about spinning here?
|
||||
} else if (ngx_errno == EAGAIN || ngx_errno == EWOULDBLOCK) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
if (size <= 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
data.connection->event_handler_(data);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
bool NgxEventConnection::WriteEvent(void* sender) {
|
||||
return WriteEvent('X' /* Anything char is fine */, sender);
|
||||
}
|
||||
|
||||
bool NgxEventConnection::WriteEvent(char type, void* sender) {
|
||||
ssize_t size = 0;
|
||||
ps_event_data data;
|
||||
|
||||
ngx_memzero(&data, sizeof(data));
|
||||
data.type = type;
|
||||
data.sender = sender;
|
||||
data.connection = this;
|
||||
|
||||
while (true) {
|
||||
size = write(pipe_write_fd_,
|
||||
static_cast<void*>(&data), sizeof(data));
|
||||
if (size == sizeof(data)) {
|
||||
return true;
|
||||
} else if (size == -1) {
|
||||
// TODO(oschaaf): should we worry about spinning here?
|
||||
if (ngx_errno == EINTR || ngx_errno == EAGAIN
|
||||
|| ngx_errno == EWOULDBLOCK) {
|
||||
continue;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
CHECK(false) << "pagespeed: unexpected return value from write(): "
|
||||
<< size;
|
||||
}
|
||||
}
|
||||
CHECK(false) << "Should not get here";
|
||||
return false;
|
||||
}
|
||||
|
||||
// Reads and processes what is available in the pipe.
|
||||
void NgxEventConnection::Drain() {
|
||||
NgxEventConnection::ReadAndNotify(pipe_read_fd_);
|
||||
}
|
||||
|
||||
void NgxEventConnection::Shutdown() {
|
||||
close(pipe_write_fd_);
|
||||
close(pipe_read_fd_);
|
||||
}
|
||||
|
||||
} // namespace net_instaweb
|
||||
84
ngx_pagespeed-latest-beta/src/ngx_event_connection.h
Normal file
84
ngx_pagespeed-latest-beta/src/ngx_event_connection.h
Normal file
@@ -0,0 +1,84 @@
|
||||
/*
|
||||
* Copyright 2014 Google Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Author: oschaaf@we-amp.com (Otto van der Schaaf)
|
||||
//
|
||||
// NgxEventConnection implements a means to send events from other threads to
|
||||
// nginx's event loop, and is implemented by a named pipe under the hood.
|
||||
// A single instance is used by NgxBaseFetch, and one instance is created per
|
||||
// NgxUrlAsyncFetcher when native fetching is on.
|
||||
|
||||
#ifndef NGX_EVENT_CONNECTION_H_
|
||||
#define NGX_EVENT_CONNECTION_H_
|
||||
|
||||
extern "C" {
|
||||
#include <ngx_http.h>
|
||||
}
|
||||
|
||||
#include <pthread.h>
|
||||
|
||||
#include "pagespeed/kernel/base/string.h"
|
||||
#include "pagespeed/kernel/http/headers.h"
|
||||
|
||||
namespace net_instaweb {
|
||||
|
||||
class NgxEventConnection;
|
||||
|
||||
// Represents a single event that can be written to or read from the pipe.
|
||||
// Technically, sender is the only data we need to send. type and connection are
|
||||
// included to provide a means to trace the events along with some more
|
||||
// info.
|
||||
typedef struct {
|
||||
char type;
|
||||
void* sender;
|
||||
NgxEventConnection* connection;
|
||||
} ps_event_data;
|
||||
|
||||
// Handler signature for receiving events
|
||||
typedef void (*callbackPtr)(const ps_event_data&);
|
||||
|
||||
// Abstracts a connection to nginx through which events can be written.
|
||||
class NgxEventConnection {
|
||||
public:
|
||||
explicit NgxEventConnection(callbackPtr handler);
|
||||
|
||||
// Creates the file descriptors and ngx_connection_t required for event
|
||||
// messaging between pagespeed and nginx.
|
||||
bool Init(ngx_cycle_t* cycle);
|
||||
// Shuts down the underlying file descriptors and connection created in Init()
|
||||
void Shutdown();
|
||||
// Constructs a ps_event_data and writes it to the underlying named pipe.
|
||||
bool WriteEvent(char type, void* sender);
|
||||
// Convenience overload for clients that have a single event type.
|
||||
bool WriteEvent(void* sender);
|
||||
// Reads and processes what is available in the named pipe's buffer.
|
||||
void Drain();
|
||||
private:
|
||||
static bool CreateNgxConnection(ngx_cycle_t* cycle, ngx_fd_t pipe_fd);
|
||||
static void ReadEventHandler(ngx_event_t* e);
|
||||
static bool ReadAndNotify(ngx_fd_t fd);
|
||||
|
||||
callbackPtr event_handler_;
|
||||
// We own these file descriptors
|
||||
ngx_fd_t pipe_write_fd_;
|
||||
ngx_fd_t pipe_read_fd_;
|
||||
|
||||
DISALLOW_COPY_AND_ASSIGN(NgxEventConnection);
|
||||
};
|
||||
|
||||
} // namespace net_instaweb
|
||||
|
||||
#endif // NGX_EVENT_CONNECTION_H_
|
||||
951
ngx_pagespeed-latest-beta/src/ngx_fetch.cc
Normal file
951
ngx_pagespeed-latest-beta/src/ngx_fetch.cc
Normal file
@@ -0,0 +1,951 @@
|
||||
/*
|
||||
* Copyright 2012 Google Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Author: x.dinic@gmail.com (Junmin Xiong)
|
||||
//
|
||||
// - The fetch is started by the main thread.
|
||||
// - Resolver event was hooked when a NgxFetch starting. It could
|
||||
// lookup the IP of the domain asynchronously from the DNS server.
|
||||
// - When NgxFetchResolveDone is called, It will create the request and the
|
||||
// connection. Add the write and read event to the epoll structure.
|
||||
// - The read handler parses the response. Add the response to the buffer at
|
||||
// last.
|
||||
|
||||
// TODO(oschaaf): Currently the first applicable connection is picked from the
|
||||
// pool when re-using connections. Perhaps it would be worth it to pick the one
|
||||
// that was active the longest time ago to keep a larger pool available.
|
||||
// TODO(oschaaf): style: reindent namespace according to google C++ style guide
|
||||
// TODO(oschaaf): Retry mechanism for failures on a re-used k-a connection.
|
||||
// Currently we don't think it's going to be an issue, see the comments at
|
||||
// https://github.com/pagespeed/ngx_pagespeed/pull/781.
|
||||
|
||||
extern "C" {
|
||||
#include <nginx.h>
|
||||
}
|
||||
|
||||
#include "ngx_fetch.h"
|
||||
|
||||
#include "base/logging.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
#include <typeinfo>
|
||||
#include <vector>
|
||||
|
||||
#include "net/instaweb/http/public/async_fetch.h"
|
||||
#include "net/instaweb/http/public/inflating_fetch.h"
|
||||
#include "net/instaweb/public/version.h"
|
||||
#include "net/instaweb/public/global_constants.h"
|
||||
#include "pagespeed/kernel/base/basictypes.h"
|
||||
#include "pagespeed/kernel/base/condvar.h"
|
||||
#include "pagespeed/kernel/base/message_handler.h"
|
||||
#include "pagespeed/kernel/base/pool.h"
|
||||
#include "pagespeed/kernel/base/pool_element.h"
|
||||
#include "pagespeed/kernel/base/scoped_ptr.h"
|
||||
#include "pagespeed/kernel/base/statistics.h"
|
||||
#include "pagespeed/kernel/base/string_writer.h"
|
||||
#include "pagespeed/kernel/base/string_util.h"
|
||||
#include "pagespeed/kernel/base/thread_system.h"
|
||||
#include "pagespeed/kernel/base/timer.h"
|
||||
#include "pagespeed/kernel/base/writer.h"
|
||||
#include "pagespeed/kernel/http/request_headers.h"
|
||||
#include "pagespeed/kernel/http/response_headers.h"
|
||||
#include "pagespeed/kernel/http/response_headers_parser.h"
|
||||
|
||||
namespace net_instaweb {
|
||||
|
||||
NgxConnection::NgxConnectionPool NgxConnection::connection_pool;
|
||||
PthreadMutex NgxConnection::connection_pool_mutex;
|
||||
// Default keepalive 60s.
|
||||
const int64 NgxConnection::keepalive_timeout_ms = 60000;
|
||||
const GoogleString NgxConnection::ka_header =
|
||||
StrCat("keep-alive ",
|
||||
Integer64ToString(NgxConnection::keepalive_timeout_ms));
|
||||
|
||||
NgxConnection::NgxConnection(MessageHandler* handler,
|
||||
int max_keepalive_requests) {
|
||||
c_ = NULL;
|
||||
max_keepalive_requests_ = max_keepalive_requests;
|
||||
handler_ = handler;
|
||||
// max_keepalive_requests specifies the number of http requests that are
|
||||
// allowed to be performed over a single connection. So, a
|
||||
// max_keepalive_requests of 1 effectively disables keepalive.
|
||||
keepalive_ = max_keepalive_requests_ > 1;
|
||||
}
|
||||
|
||||
NgxConnection::~NgxConnection() {
|
||||
CHECK(c_ == NULL) << "NgxConnection: Underlying connection should be NULL";
|
||||
}
|
||||
|
||||
void NgxConnection::Terminate() {
|
||||
for (NgxConnectionPool::iterator p = connection_pool.begin();
|
||||
p != connection_pool.end(); ++p) {
|
||||
NgxConnection* nc = *p;
|
||||
ngx_close_connection(nc->c_);
|
||||
nc->c_ = NULL;
|
||||
delete nc;
|
||||
}
|
||||
connection_pool.Clear();
|
||||
}
|
||||
|
||||
NgxConnection* NgxConnection::Connect(ngx_peer_connection_t* pc,
|
||||
MessageHandler* handler,
|
||||
int max_keepalive_requests) {
|
||||
NgxConnection* nc;
|
||||
{
|
||||
ScopedMutex lock(&NgxConnection::connection_pool_mutex);
|
||||
|
||||
for (NgxConnectionPool::iterator p = connection_pool.begin();
|
||||
p != connection_pool.end(); ++p) {
|
||||
nc = *p;
|
||||
|
||||
if (ngx_memn2cmp(static_cast<u_char*>(nc->sockaddr_),
|
||||
reinterpret_cast<u_char*>(pc->sockaddr),
|
||||
nc->socklen_, pc->socklen) == 0) {
|
||||
CHECK(nc->c_->idle) << "Pool should only contain idle connections!";
|
||||
|
||||
nc->c_->idle = 0;
|
||||
nc->c_->log = pc->log;
|
||||
nc->c_->read->log = pc->log;
|
||||
nc->c_->write->log = pc->log;
|
||||
if (nc->c_->pool != NULL) {
|
||||
nc->c_->pool->log = pc->log;
|
||||
}
|
||||
|
||||
if (nc->c_->read->timer_set) {
|
||||
ngx_del_timer(nc->c_->read);
|
||||
}
|
||||
connection_pool.Remove(nc);
|
||||
|
||||
ngx_log_error(NGX_LOG_DEBUG, pc->log, 0,
|
||||
"NgxFetch: re-using connection %p (pool size: %l)",
|
||||
nc, connection_pool.size());
|
||||
return nc;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int rc = ngx_event_connect_peer(pc);
|
||||
if (rc == NGX_ERROR || rc == NGX_DECLINED || rc == NGX_BUSY) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// NgxConnection deletes itself if NgxConnection::Close()
|
||||
nc = new NgxConnection(handler, max_keepalive_requests);
|
||||
nc->SetSock(reinterpret_cast<u_char*>(pc->sockaddr), pc->socklen);
|
||||
nc->c_ = pc->connection;
|
||||
return nc;
|
||||
}
|
||||
|
||||
void NgxConnection::Close() {
|
||||
bool removed_from_pool = false;
|
||||
|
||||
{
|
||||
ScopedMutex lock(&NgxConnection::connection_pool_mutex);
|
||||
for (NgxConnectionPool::iterator p = connection_pool.begin();
|
||||
p != connection_pool.end(); ++p) {
|
||||
if (*p == this) {
|
||||
// When we get here, that means that the connection either has timed
|
||||
// out or has been closed remotely.
|
||||
connection_pool.Remove(this);
|
||||
ngx_log_error(NGX_LOG_DEBUG, c_->log, 0,
|
||||
"NgxFetch: removed connection %p (pool size: %l)",
|
||||
this, connection_pool.size());
|
||||
removed_from_pool = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
max_keepalive_requests_--;
|
||||
|
||||
if (c_->read->timer_set) {
|
||||
ngx_del_timer(c_->read);
|
||||
}
|
||||
|
||||
if (c_->write->timer_set) {
|
||||
ngx_del_timer(c_->write);
|
||||
}
|
||||
|
||||
if (!keepalive_ || max_keepalive_requests_ <= 0 || removed_from_pool) {
|
||||
ngx_close_connection(c_);
|
||||
c_ = NULL;
|
||||
delete this;
|
||||
return;
|
||||
}
|
||||
|
||||
ngx_add_timer(c_->read, static_cast<ngx_msec_t>(
|
||||
NgxConnection::keepalive_timeout_ms));
|
||||
|
||||
c_->data = this;
|
||||
c_->read->handler = NgxConnection::IdleReadHandler;
|
||||
c_->write->handler = NgxConnection::IdleWriteHandler;
|
||||
c_->idle = 1;
|
||||
|
||||
// This connection should not be associated with current fetch.
|
||||
c_->log = ngx_cycle->log;
|
||||
c_->read->log = ngx_cycle->log;
|
||||
c_->write->log = ngx_cycle->log;
|
||||
if (c_->pool != NULL) {
|
||||
c_->pool->log = ngx_cycle->log;
|
||||
}
|
||||
|
||||
// Allow this connection to be re-used, by adding it to the connection pool.
|
||||
{
|
||||
ScopedMutex lock(&NgxConnection::connection_pool_mutex);
|
||||
connection_pool.Add(this);
|
||||
ngx_log_error(NGX_LOG_DEBUG, c_->log, 0,
|
||||
"NgxFetch: Added connection %p (pool size: %l - "
|
||||
" max_keepalive_requests_ %d)",
|
||||
this, connection_pool.size(), max_keepalive_requests_);
|
||||
}
|
||||
}
|
||||
|
||||
void NgxConnection::IdleWriteHandler(ngx_event_t* ev) {
|
||||
ngx_connection_t* c = static_cast<ngx_connection_t*>(ev->data);
|
||||
u_char buf[1];
|
||||
int n = c->recv(c, buf, 1);
|
||||
if (c->write->timedout) {
|
||||
DCHECK(false) << "NgxFetch: write timeout not expected." << n;
|
||||
}
|
||||
if (n == NGX_AGAIN) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
void NgxConnection::IdleReadHandler(ngx_event_t* ev) {
|
||||
ngx_connection_t* c = static_cast<ngx_connection_t*>(ev->data);
|
||||
NgxConnection* nc = static_cast<NgxConnection*>(c->data);
|
||||
|
||||
if (c->read->timedout) {
|
||||
nc->set_keepalive(false);
|
||||
nc->Close();
|
||||
return;
|
||||
}
|
||||
|
||||
char buf[1];
|
||||
int n;
|
||||
|
||||
// not a timeout event, we should check connection
|
||||
n = recv(c->fd, buf, 1, MSG_PEEK);
|
||||
if (n == -1 && ngx_socket_errno == NGX_EAGAIN) {
|
||||
if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
|
||||
nc->set_keepalive(false);
|
||||
nc->Close();
|
||||
return;
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
nc->set_keepalive(false);
|
||||
nc->Close();
|
||||
}
|
||||
|
||||
NgxFetch::NgxFetch(const GoogleString& url,
|
||||
AsyncFetch* async_fetch,
|
||||
MessageHandler* message_handler,
|
||||
ngx_log_t* log)
|
||||
: str_url_(url),
|
||||
fetcher_(NULL),
|
||||
async_fetch_(async_fetch),
|
||||
parser_(async_fetch->response_headers()),
|
||||
message_handler_(message_handler),
|
||||
bytes_received_(0),
|
||||
fetch_start_ms_(0),
|
||||
fetch_end_ms_(0),
|
||||
done_(false),
|
||||
content_length_(-1),
|
||||
content_length_known_(false),
|
||||
resolver_ctx_(NULL) {
|
||||
ngx_memzero(&url_, sizeof(url_));
|
||||
log_ = log;
|
||||
pool_ = NULL;
|
||||
timeout_event_ = NULL;
|
||||
connection_ = NULL;
|
||||
}
|
||||
|
||||
NgxFetch::~NgxFetch() {
|
||||
if (timeout_event_ != NULL && timeout_event_->timer_set) {
|
||||
ngx_del_timer(timeout_event_);
|
||||
}
|
||||
if (connection_ != NULL) {
|
||||
connection_->Close();
|
||||
connection_ = NULL;
|
||||
}
|
||||
if (pool_ != NULL) {
|
||||
ngx_destroy_pool(pool_);
|
||||
pool_ = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
// This function is called by NgxUrlAsyncFetcher::StartFetch.
|
||||
bool NgxFetch::Start(NgxUrlAsyncFetcher* fetcher) {
|
||||
fetcher_ = fetcher;
|
||||
bool ok = Init();
|
||||
if (ok) {
|
||||
ngx_log_error(NGX_LOG_DEBUG, log_, 0, "NgxFetch %p: initialized",
|
||||
this);
|
||||
} // else Init() will have emitted a reason
|
||||
return ok;
|
||||
}
|
||||
|
||||
// Create the pool, parse the url, add the timeout event and
|
||||
// hook the DNS resolver if needed. Else we connect directly.
|
||||
// When this returns false, our caller (NgxUrlAsyncFetcher::StartFetch)
|
||||
// will call fetch->CallbackDone()
|
||||
bool NgxFetch::Init() {
|
||||
pool_ = ngx_create_pool(12288, log_);
|
||||
if (pool_ == NULL) {
|
||||
message_handler_->Message(kError, "NgxFetch: ngx_create_pool failed");
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!ParseUrl()) {
|
||||
message_handler_->Message(kError,
|
||||
"NgxFetch: ParseUrl() failed for [%s]:%s",
|
||||
str_url_.c_str(), url_.err);
|
||||
return false;
|
||||
}
|
||||
|
||||
timeout_event_ = static_cast<ngx_event_t*>(
|
||||
ngx_pcalloc(pool_, sizeof(ngx_event_t)));
|
||||
if (timeout_event_ == NULL) {
|
||||
message_handler_->Message(kError,
|
||||
"NgxFetch: ngx_pcalloc failed for timeout");
|
||||
return false;
|
||||
}
|
||||
|
||||
timeout_event_->data = this;
|
||||
timeout_event_->handler = NgxFetch::TimeoutHandler;
|
||||
timeout_event_->log = log_;
|
||||
|
||||
ngx_add_timer(timeout_event_, fetcher_->fetch_timeout_);
|
||||
r_ = static_cast<ngx_http_request_t*>(
|
||||
ngx_pcalloc(pool_, sizeof(ngx_http_request_t)));
|
||||
|
||||
if (r_ == NULL) {
|
||||
message_handler_->Message(kError,
|
||||
"NgxFetch: ngx_pcalloc failed for timer");
|
||||
return false;
|
||||
}
|
||||
status_ = static_cast<ngx_http_status_t*>(
|
||||
ngx_pcalloc(pool_, sizeof(ngx_http_status_t)));
|
||||
|
||||
if (status_ == NULL) {
|
||||
message_handler_->Message(kError,
|
||||
"NgxFetch: ngx_pcalloc failed for status");
|
||||
return false;
|
||||
}
|
||||
|
||||
// The host is either a domain name or an IP address. First check
|
||||
// if it's a valid IP address and only if that fails fall back to
|
||||
// using the DNS resolver.
|
||||
|
||||
// Maybe we have a Proxy.
|
||||
ngx_url_t* tmp_url = &url_;
|
||||
if (fetcher_->proxy_.url.len != 0) {
|
||||
tmp_url = &fetcher_->proxy_;
|
||||
}
|
||||
|
||||
GoogleString s_ipaddress(reinterpret_cast<char*>(tmp_url->host.data),
|
||||
tmp_url->host.len);
|
||||
ngx_memzero(&sin_, sizeof(sin_));
|
||||
sin_.sin_family = AF_INET;
|
||||
sin_.sin_port = htons(tmp_url->port);
|
||||
sin_.sin_addr.s_addr = inet_addr(s_ipaddress.c_str());
|
||||
|
||||
if (sin_.sin_addr.s_addr == INADDR_NONE) {
|
||||
// inet_addr returned INADDR_NONE, which means the hostname
|
||||
// isn't a valid IP address. Check DNS.
|
||||
ngx_resolver_ctx_t temp;
|
||||
temp.name.data = tmp_url->host.data;
|
||||
temp.name.len = tmp_url->host.len;
|
||||
resolver_ctx_ = ngx_resolve_start(fetcher_->resolver_, &temp);
|
||||
if (resolver_ctx_ == NULL || resolver_ctx_ == NGX_NO_RESOLVER) {
|
||||
// TODO(oschaaf): this spams the log, but is useful in the fetcher's
|
||||
// current state
|
||||
message_handler_->Message(
|
||||
kError, "NgxFetch: Couldn't start resolving, "
|
||||
"is there a proper resolver configured in nginx.conf?");
|
||||
return false;
|
||||
} else {
|
||||
ngx_log_error(NGX_LOG_DEBUG, log_, 0,
|
||||
"NgxFetch %p: start resolve for: %s",
|
||||
this, s_ipaddress.c_str());
|
||||
}
|
||||
|
||||
resolver_ctx_->data = this;
|
||||
resolver_ctx_->name.data = tmp_url->host.data;
|
||||
resolver_ctx_->name.len = tmp_url->host.len;
|
||||
|
||||
#if (nginx_version < 1005008)
|
||||
resolver_ctx_->type = NGX_RESOLVE_A;
|
||||
#endif
|
||||
|
||||
resolver_ctx_->handler = NgxFetch::ResolveDoneHandler;
|
||||
resolver_ctx_->timeout = fetcher_->resolver_timeout_;
|
||||
|
||||
if (ngx_resolve_name(resolver_ctx_) != NGX_OK) {
|
||||
message_handler_->Message(kWarning,
|
||||
"NgxFetch: ngx_resolve_name failed");
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
if (InitRequest() != NGX_OK) {
|
||||
message_handler()->Message(kError, "NgxFetch: InitRequest failed");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
const char* NgxFetch::str_url() {
|
||||
return str_url_.c_str();
|
||||
}
|
||||
|
||||
// This function should be called only once. The only argument is sucess or
|
||||
// not.
|
||||
void NgxFetch::CallbackDone(bool success) {
|
||||
ngx_log_error(NGX_LOG_DEBUG, log_, 0, "NgxFetch %p: CallbackDone: %s",
|
||||
this, success ? "OK":"FAIL");
|
||||
|
||||
if (async_fetch_ == NULL) {
|
||||
LOG(FATAL)
|
||||
<< "BUG: NgxFetch callback called more than once on same fetch"
|
||||
<< str_url_.c_str() << "(" << this << ").Please report this at"
|
||||
<< "https://groups.google.com/forum/#!forum/ngx-pagespeed-discuss";
|
||||
return;
|
||||
}
|
||||
|
||||
release_resolver();
|
||||
|
||||
if (timeout_event_ && timeout_event_->timer_set) {
|
||||
ngx_del_timer(timeout_event_);
|
||||
timeout_event_ = NULL;
|
||||
}
|
||||
|
||||
if (connection_ != NULL) {
|
||||
// Connection will be re-used only on responses that specify
|
||||
// 'Connection: keep-alive' in their headers.
|
||||
bool keepalive = false;
|
||||
|
||||
if (success) {
|
||||
ConstStringStarVector v;
|
||||
if (async_fetch_->response_headers()->Lookup(
|
||||
StringPiece(HttpAttributes::kConnection), &v)) {
|
||||
for (size_t i = 0; i < v.size(); i++) {
|
||||
if (*v[i] == "keep-alive") {
|
||||
keepalive = true;
|
||||
break;
|
||||
} else if (*v[i] == "close") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
ngx_log_error(NGX_LOG_DEBUG, log_, 0,
|
||||
"NgxFetch %p: connection %p attempt keep-alive: %s",
|
||||
this, connection_, keepalive ? "Yes":"No");
|
||||
}
|
||||
|
||||
connection_->set_keepalive(keepalive);
|
||||
connection_->Close();
|
||||
connection_ = NULL;
|
||||
}
|
||||
|
||||
if (fetcher_ != NULL) {
|
||||
if (fetcher_->track_original_content_length()
|
||||
&& async_fetch_->response_headers()->Has(
|
||||
HttpAttributes::kXOriginalContentLength)) {
|
||||
async_fetch_->extra_response_headers()->SetOriginalContentLength(
|
||||
bytes_received_);
|
||||
}
|
||||
fetcher_->FetchComplete(this);
|
||||
}
|
||||
async_fetch_->Done(success);
|
||||
async_fetch_ = NULL;
|
||||
}
|
||||
|
||||
size_t NgxFetch::bytes_received() {
|
||||
return bytes_received_;
|
||||
}
|
||||
void NgxFetch::bytes_received_add(int64 x) {
|
||||
bytes_received_ += x;
|
||||
}
|
||||
|
||||
int64 NgxFetch::fetch_start_ms() {
|
||||
return fetch_start_ms_;
|
||||
}
|
||||
|
||||
void NgxFetch::set_fetch_start_ms(int64 start_ms) {
|
||||
fetch_start_ms_ = start_ms;
|
||||
}
|
||||
|
||||
int64 NgxFetch::fetch_end_ms() {
|
||||
return fetch_end_ms_;
|
||||
}
|
||||
|
||||
void NgxFetch::set_fetch_end_ms(int64 end_ms) {
|
||||
fetch_end_ms_ = end_ms;
|
||||
}
|
||||
|
||||
MessageHandler* NgxFetch::message_handler() {
|
||||
return message_handler_;
|
||||
}
|
||||
|
||||
bool NgxFetch::ParseUrl() {
|
||||
url_.url.len = str_url_.length();
|
||||
url_.url.data = static_cast<u_char*>(ngx_palloc(pool_, url_.url.len));
|
||||
if (url_.url.data == NULL) {
|
||||
return false;
|
||||
}
|
||||
str_url_.copy(reinterpret_cast<char*>(url_.url.data), str_url_.length(), 0);
|
||||
|
||||
return NgxUrlAsyncFetcher::ParseUrl(&url_, pool_);
|
||||
}
|
||||
|
||||
// Issue a request after the resolver is done
|
||||
void NgxFetch::ResolveDoneHandler(ngx_resolver_ctx_t* resolver_ctx) {
|
||||
NgxFetch* fetch = static_cast<NgxFetch*>(resolver_ctx->data);
|
||||
NgxUrlAsyncFetcher* fetcher = fetch->fetcher_;
|
||||
|
||||
if (resolver_ctx->state != NGX_OK) {
|
||||
if (fetch->timeout_event() != NULL && fetch->timeout_event()->timer_set) {
|
||||
ngx_del_timer(fetch->timeout_event());
|
||||
fetch->set_timeout_event(NULL);
|
||||
}
|
||||
fetch->message_handler()->Message(
|
||||
kWarning, "NgxFetch %p: failed to resolve host [%.*s]", fetch,
|
||||
static_cast<int>(resolver_ctx->name.len), resolver_ctx->name.data);
|
||||
fetch->CallbackDone(false);
|
||||
return;
|
||||
}
|
||||
|
||||
ngx_uint_t i;
|
||||
// Find the first ipv4 address. We don't support ipv6 yet.
|
||||
for (i = 0; i < resolver_ctx->naddrs; i++) {
|
||||
// Old versions of nginx and tengine have a different definition of addrs,
|
||||
// work around to make sure we are using the right type (ngx_addr_t*).
|
||||
ngx_addr_t* ngx_addrs = reinterpret_cast<ngx_addr_t*>(resolver_ctx->addrs);
|
||||
if (typeid(*ngx_addrs) == typeid(*resolver_ctx->addrs)) {
|
||||
if (reinterpret_cast<struct sockaddr_in*>(ngx_addrs[i].sockaddr)
|
||||
->sin_family == AF_INET) {
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
// We're using an old version that uses in_addr_t* for addrs.
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// If no suitable ipv4 address was found, we fail.
|
||||
if (i == resolver_ctx->naddrs) {
|
||||
if (fetch->timeout_event() != NULL && fetch->timeout_event()->timer_set) {
|
||||
ngx_del_timer(fetch->timeout_event());
|
||||
fetch->set_timeout_event(NULL);
|
||||
}
|
||||
fetch->message_handler()->Message(
|
||||
kWarning, "NgxFetch %p: no suitable address for host [%.*s]", fetch,
|
||||
static_cast<int>(resolver_ctx->name.len), resolver_ctx->name.data);
|
||||
fetch->CallbackDone(false);
|
||||
}
|
||||
|
||||
ngx_memzero(&fetch->sin_, sizeof(fetch->sin_));
|
||||
|
||||
#if (nginx_version < 1005008)
|
||||
fetch->sin_.sin_addr.s_addr = resolver_ctx->addrs[i];
|
||||
#else
|
||||
struct sockaddr_in* sin;
|
||||
|
||||
sin = reinterpret_cast<struct sockaddr_in*>(
|
||||
resolver_ctx->addrs[i].sockaddr);
|
||||
|
||||
fetch->sin_.sin_family = sin->sin_family;
|
||||
fetch->sin_.sin_addr.s_addr = sin->sin_addr.s_addr;
|
||||
#endif
|
||||
|
||||
fetch->sin_.sin_family = AF_INET;
|
||||
fetch->sin_.sin_port = htons(fetch->url_.port);
|
||||
|
||||
// Maybe we have Proxy
|
||||
if (0 != fetcher->proxy_.url.len) {
|
||||
fetch->sin_.sin_port = htons(fetcher->proxy_.port);
|
||||
}
|
||||
|
||||
char* ip_address = inet_ntoa(fetch->sin_.sin_addr);
|
||||
|
||||
ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
|
||||
"NgxFetch %p: Resolved host [%V] to [%s]", fetch,
|
||||
&resolver_ctx->name, ip_address);
|
||||
|
||||
fetch->release_resolver();
|
||||
|
||||
if (fetch->InitRequest() != NGX_OK) {
|
||||
fetch->message_handler()->Message(kError, "NgxFetch: InitRequest failed");
|
||||
fetch->CallbackDone(false);
|
||||
}
|
||||
}
|
||||
|
||||
// Prepare the request data for this fetch, and hook the write event.
|
||||
int NgxFetch::InitRequest() {
|
||||
in_ = ngx_create_temp_buf(pool_, 4096);
|
||||
if (in_ == NULL) {
|
||||
return NGX_ERROR;
|
||||
}
|
||||
|
||||
FixUserAgent();
|
||||
|
||||
RequestHeaders* request_headers = async_fetch_->request_headers();
|
||||
ConstStringStarVector v;
|
||||
size_t size = 0;
|
||||
bool have_host = false;
|
||||
GoogleString port;
|
||||
|
||||
response_handler = NgxFetch::HandleStatusLine;
|
||||
int rc = Connect();
|
||||
if (rc == NGX_AGAIN || rc == NGX_OK) {
|
||||
if (connection_->keepalive()) {
|
||||
request_headers->Add(HttpAttributes::kConnection,
|
||||
NgxConnection::ka_header);
|
||||
}
|
||||
const char* method = request_headers->method_string();
|
||||
size_t method_len = strlen(method);
|
||||
|
||||
size = (method_len +
|
||||
1 /* for the space */ +
|
||||
url_.uri.len +
|
||||
sizeof(" HTTP/1.0\r\n") - 1);
|
||||
|
||||
for (int i = 0; i < request_headers->NumAttributes(); i++) {
|
||||
// if no explicit host header is given in the request headers,
|
||||
// we need to derive it from the url.
|
||||
if (StringCaseEqual(request_headers->Name(i), "Host")) {
|
||||
have_host = true;
|
||||
}
|
||||
|
||||
// name: value\r\n
|
||||
size += request_headers->Name(i).length()
|
||||
+ request_headers->Value(i).length() + 4; // 4 for ": \r\n"
|
||||
}
|
||||
|
||||
if (!have_host) {
|
||||
port = StrCat(":", IntegerToString(url_.port));
|
||||
// for "Host: " + host + ":" + port + "\r\n"
|
||||
size += url_.host.len + 8 + port.size();
|
||||
}
|
||||
|
||||
size += 2; // "\r\n";
|
||||
out_ = ngx_create_temp_buf(pool_, size);
|
||||
|
||||
if (out_ == NULL) {
|
||||
return NGX_ERROR;
|
||||
}
|
||||
|
||||
out_->last = ngx_cpymem(out_->last, method, method_len);
|
||||
out_->last = ngx_cpymem(out_->last, " ", 1);
|
||||
out_->last = ngx_cpymem(out_->last, url_.uri.data, url_.uri.len);
|
||||
out_->last = ngx_cpymem(out_->last, " HTTP/1.0\r\n", 11);
|
||||
|
||||
if (!have_host) {
|
||||
out_->last = ngx_cpymem(out_->last, "Host: ", 6);
|
||||
out_->last = ngx_cpymem(out_->last, url_.host.data, url_.host.len);
|
||||
out_->last = ngx_cpymem(out_->last, port.c_str(), port.size());
|
||||
out_->last = ngx_cpymem(out_->last, "\r\n", 2);
|
||||
}
|
||||
|
||||
for (int i = 0; i < request_headers->NumAttributes(); i++) {
|
||||
const GoogleString& name = request_headers->Name(i);
|
||||
const GoogleString& value = request_headers->Value(i);
|
||||
out_->last = ngx_cpymem(out_->last, name.c_str(), name.length());
|
||||
*(out_->last++) = ':';
|
||||
*(out_->last++) = ' ';
|
||||
out_->last = ngx_cpymem(out_->last, value.c_str(), value.length());
|
||||
*(out_->last++) = CR;
|
||||
*(out_->last++) = LF;
|
||||
}
|
||||
*(out_->last++) = CR;
|
||||
*(out_->last++) = LF;
|
||||
if (rc == NGX_AGAIN) {
|
||||
return NGX_OK;
|
||||
}
|
||||
} else if (rc < NGX_OK) {
|
||||
return rc;
|
||||
}
|
||||
CHECK(rc == NGX_OK);
|
||||
NgxFetch::ConnectionWriteHandler(connection_->c_->write);
|
||||
return NGX_OK;
|
||||
}
|
||||
|
||||
int NgxFetch::Connect() {
|
||||
ngx_peer_connection_t pc;
|
||||
ngx_memzero(&pc, sizeof(pc));
|
||||
pc.sockaddr = (struct sockaddr*)&sin_;
|
||||
pc.socklen = sizeof(struct sockaddr_in);
|
||||
pc.name = &url_.host;
|
||||
|
||||
// get callback is dummy function, it just returns NGX_OK
|
||||
pc.get = ngx_event_get_peer;
|
||||
pc.log_error = NGX_ERROR_ERR;
|
||||
pc.log = fetcher_->log_;
|
||||
pc.rcvbuf = -1;
|
||||
|
||||
|
||||
connection_ = NgxConnection::Connect(&pc, message_handler(),
|
||||
fetcher_->max_keepalive_requests_);
|
||||
ngx_log_error(NGX_LOG_DEBUG, fetcher_->log_, 0,
|
||||
"NgxFetch %p Connect() connection %p for [%s]",
|
||||
this, connection_, str_url());
|
||||
|
||||
if (connection_ == NULL) {
|
||||
return NGX_ERROR;
|
||||
}
|
||||
|
||||
connection_->c_->write->handler = NgxFetch::ConnectionWriteHandler;
|
||||
connection_->c_->read->handler = NgxFetch::ConnectionReadHandler;
|
||||
connection_->c_->data = this;
|
||||
|
||||
// Timer set in Init() is still in effect.
|
||||
return NGX_OK;
|
||||
}
|
||||
|
||||
// When the fetch sends the request completely, it will hook the read event,
|
||||
// and prepare to parse the response. Timer set in Init() is still in effect.
|
||||
void NgxFetch::ConnectionWriteHandler(ngx_event_t* wev) {
|
||||
ngx_connection_t* c = static_cast<ngx_connection_t*>(wev->data);
|
||||
NgxFetch* fetch = static_cast<NgxFetch*>(c->data);
|
||||
ngx_buf_t* out = fetch->out_;
|
||||
bool ok = true;
|
||||
while (wev->ready && out->pos < out->last) {
|
||||
int n = c->send(c, out->pos, out->last - out->pos);
|
||||
ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
|
||||
"NgxFetch %p: ConnectionWriteHandler "
|
||||
"send result %d", fetch, n);
|
||||
|
||||
if (n >= 0) {
|
||||
out->pos += n;
|
||||
} else if (n == NGX_AGAIN) {
|
||||
break;
|
||||
} else {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (ok) {
|
||||
if (out->pos == out->last) {
|
||||
ok = ngx_handle_read_event(c->read, 0) == NGX_OK;
|
||||
} else {
|
||||
ok = ngx_handle_write_event(c->write, 0) == NGX_OK;
|
||||
}
|
||||
}
|
||||
|
||||
if (!ok) {
|
||||
fetch->message_handler()->Message(
|
||||
kWarning, "NgxFetch %p: failed to hook next event", fetch);
|
||||
c->error = 1;
|
||||
fetch->CallbackDone(false);
|
||||
}
|
||||
}
|
||||
|
||||
// Timer set in Init() is still in effect.
|
||||
void NgxFetch::ConnectionReadHandler(ngx_event_t* rev) {
|
||||
ngx_connection_t* c = static_cast<ngx_connection_t*>(rev->data);
|
||||
NgxFetch* fetch = static_cast<NgxFetch*>(c->data);
|
||||
bool ok = true;
|
||||
|
||||
while(rev->ready) {
|
||||
int n = c->recv(
|
||||
c, fetch->in_->start, fetch->in_->end - fetch->in_->start);
|
||||
|
||||
ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
|
||||
"NgxFetch %p: ConnectionReadHandler "
|
||||
"recv result %d", fetch, n);
|
||||
|
||||
if (n == NGX_AGAIN) {
|
||||
break;
|
||||
} else if (n == 0) {
|
||||
// If the content length was not known, we assume that we have read
|
||||
// all if we at least parsed the headers.
|
||||
// If we do know the content length, having a mismatch on the bytes read
|
||||
// will be interpreted as an error.
|
||||
ok = (fetch->content_length_known_ && fetch->content_length_ == fetch->bytes_received_)
|
||||
|| fetch->parser_.headers_complete();
|
||||
fetch->done_ = true;
|
||||
break;
|
||||
} else if (n > 0) {
|
||||
fetch->in_->pos = fetch->in_->start;
|
||||
fetch->in_->last = fetch->in_->start + n;
|
||||
ok = fetch->response_handler(c);
|
||||
if (fetch->done_ || !ok) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!ok) {
|
||||
fetch->CallbackDone(false);
|
||||
} else if (fetch->done_) {
|
||||
fetch->CallbackDone(true);
|
||||
} else if (ngx_handle_read_event(rev, 0) != NGX_OK) {
|
||||
fetch->CallbackDone(false);
|
||||
}
|
||||
}
|
||||
|
||||
// Parse the status line: "HTTP/1.1 200 OK\r\n"
|
||||
bool NgxFetch::HandleStatusLine(ngx_connection_t* c) {
|
||||
NgxFetch* fetch = static_cast<NgxFetch*>(c->data);
|
||||
ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
|
||||
"NgxFetch %p: Handle status line", fetch);
|
||||
|
||||
// This function only works after Nginx-1.1.4. Before nginx-1.1.4,
|
||||
// ngx_http_parse_status_line didn't save http_version.
|
||||
ngx_int_t n = ngx_http_parse_status_line(fetch->r_, fetch->in_,
|
||||
fetch->status_);
|
||||
if (n == NGX_ERROR) { // parse status line error
|
||||
fetch->message_handler()->Message(
|
||||
kWarning, "NgxFetch: failed to parse status line");
|
||||
return false;
|
||||
} else if (n == NGX_AGAIN) { // not completed
|
||||
return true;
|
||||
}
|
||||
|
||||
ResponseHeaders* response_headers =
|
||||
fetch->async_fetch_->response_headers();
|
||||
response_headers->SetStatusAndReason(
|
||||
static_cast<HttpStatus::Code>(fetch->get_status_code()));
|
||||
response_headers->set_major_version(fetch->get_major_version());
|
||||
response_headers->set_minor_version(fetch->get_minor_version());
|
||||
|
||||
fetch->in_->pos += n;
|
||||
fetch->set_response_handler(NgxFetch::HandleHeader);
|
||||
if ((fetch->in_->last - fetch->in_->pos) > 0) {
|
||||
return fetch->response_handler(c);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// Parse the HTTP headers
|
||||
bool NgxFetch::HandleHeader(ngx_connection_t* c) {
|
||||
NgxFetch* fetch = static_cast<NgxFetch*>(c->data);
|
||||
char* data = reinterpret_cast<char*>(fetch->in_->pos);
|
||||
size_t size = fetch->in_->last - fetch->in_->pos;
|
||||
size_t n = fetch->parser_.ParseChunk(StringPiece(data, size),
|
||||
fetch->message_handler_);
|
||||
|
||||
ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
|
||||
"NgxFetch %p: Handle headers", fetch);
|
||||
|
||||
if (n > size) {
|
||||
return false;
|
||||
} else if (fetch->parser_.headers_complete()) {
|
||||
// TODO(oschaaf): We should also check if the request method was HEAD
|
||||
// - but I don't think PSOL uses that at this point.
|
||||
if (fetch->get_status_code() == 304 || fetch->get_status_code() == 204) {
|
||||
fetch->done_ = true;
|
||||
} else if (fetch->async_fetch_->response_headers()->FindContentLength(
|
||||
&fetch->content_length_)) {
|
||||
if (fetch->content_length_ < 0) {
|
||||
fetch->message_handler_->Message(
|
||||
kError, "Negative content-length in response header");
|
||||
return false;
|
||||
} else {
|
||||
fetch->content_length_known_ = true;
|
||||
if (fetch->content_length_ == 0) {
|
||||
fetch->done_ = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (fetch->fetcher_->track_original_content_length()
|
||||
&& fetch->content_length_known_) {
|
||||
fetch->async_fetch_->response_headers()->SetOriginalContentLength(
|
||||
fetch->content_length_);
|
||||
}
|
||||
|
||||
fetch->in_->pos += n;
|
||||
if (!fetch->done_) {
|
||||
fetch->set_response_handler(NgxFetch::HandleBody);
|
||||
if ((fetch->in_->last - fetch->in_->pos) > 0) {
|
||||
return fetch->response_handler(c);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
fetch->in_->pos += n;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// Read the response body
|
||||
bool NgxFetch::HandleBody(ngx_connection_t* c) {
|
||||
NgxFetch* fetch = static_cast<NgxFetch*>(c->data);
|
||||
char* data = reinterpret_cast<char*>(fetch->in_->pos);
|
||||
size_t size = fetch->in_->last - fetch->in_->pos;
|
||||
|
||||
fetch->bytes_received_add(size);
|
||||
|
||||
ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
|
||||
"NgxFetch %p: Handle body (%d bytes)", fetch, size);
|
||||
|
||||
if ( fetch->async_fetch_->Write(StringPiece(data, size),
|
||||
fetch->message_handler()) ) {
|
||||
if (fetch->bytes_received_ == fetch->content_length_) {
|
||||
fetch->done_ = true;
|
||||
}
|
||||
fetch->in_->pos += size;
|
||||
} else {
|
||||
ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
|
||||
"NgxFetch %p: async fetch write failure", fetch);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void NgxFetch::TimeoutHandler(ngx_event_t* tev) {
|
||||
NgxFetch* fetch = static_cast<NgxFetch*>(tev->data);
|
||||
ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
|
||||
"NgxFetch %p: TimeoutHandler called", fetch);
|
||||
fetch->CallbackDone(false);
|
||||
}
|
||||
|
||||
void NgxFetch::FixUserAgent() {
|
||||
GoogleString user_agent;
|
||||
ConstStringStarVector v;
|
||||
RequestHeaders* request_headers = async_fetch_->request_headers();
|
||||
if (request_headers->Lookup(HttpAttributes::kUserAgent, &v)) {
|
||||
for (size_t i = 0, n = v.size(); i < n; i++) {
|
||||
if (i != 0) {
|
||||
user_agent += " ";
|
||||
}
|
||||
|
||||
if (v[i] != NULL) {
|
||||
user_agent += *(v[i]);
|
||||
}
|
||||
}
|
||||
request_headers->RemoveAll(HttpAttributes::kUserAgent);
|
||||
}
|
||||
if (user_agent.empty()) {
|
||||
user_agent += "NgxNativeFetcher";
|
||||
}
|
||||
GoogleString version = StrCat(
|
||||
" (", kModPagespeedSubrequestUserAgent,
|
||||
"/" MOD_PAGESPEED_VERSION_STRING "-" LASTCHANGE_STRING ")");
|
||||
if (!StringPiece(user_agent).ends_with(version)) {
|
||||
user_agent += version;
|
||||
}
|
||||
request_headers->Add(HttpAttributes::kUserAgent, user_agent);
|
||||
}
|
||||
|
||||
} // namespace net_instaweb
|
||||
210
ngx_pagespeed-latest-beta/src/ngx_fetch.h
Normal file
210
ngx_pagespeed-latest-beta/src/ngx_fetch.h
Normal file
@@ -0,0 +1,210 @@
|
||||
/*
|
||||
* Copyright 2012 Google Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Author: x.dinic@gmail.com (Junmin Xiong)
|
||||
//
|
||||
// PageSpeed needs some way to talk to the internet and request resources. For
|
||||
// example, if it's optimizing www.example.com/index.html and it sees html with
|
||||
// <img src="//images.example.com/cat.jpg"> and images.example.com is authorized
|
||||
// for rewriting in the config, then it needs to fetch cat.jpg from
|
||||
// images.example.com and optimize it. In apache (always) and nginx (by
|
||||
// default) we use a fetcher called "serf". This works fine, but it does run
|
||||
// its own event loop. To be more efficient, this is a "native" fetcher that
|
||||
// uses nginx's event loop.
|
||||
//
|
||||
// The fetch is started by the main thread. It will fetch the remote resource
|
||||
// from the specific url asynchronously.
|
||||
|
||||
#ifndef NET_INSTAWEB_NGX_FETCH_H_
|
||||
#define NET_INSTAWEB_NGX_FETCH_H_
|
||||
|
||||
extern "C" {
|
||||
#include <ngx_config.h>
|
||||
#include <ngx_core.h>
|
||||
#include <ngx_http.h>
|
||||
}
|
||||
|
||||
#include "ngx_url_async_fetcher.h"
|
||||
#include <vector>
|
||||
#include "net/instaweb/http/public/url_async_fetcher.h"
|
||||
#include "pagespeed/kernel/base/basictypes.h"
|
||||
#include "pagespeed/kernel/base/pool.h"
|
||||
#include "pagespeed/kernel/base/string.h"
|
||||
#include "pagespeed/kernel/http/response_headers.h"
|
||||
#include "pagespeed/kernel/http/response_headers_parser.h"
|
||||
#include "pagespeed/kernel/thread/pthread_mutex.h"
|
||||
|
||||
|
||||
namespace net_instaweb {
|
||||
|
||||
typedef bool (*response_handler_pt)(ngx_connection_t* c);
|
||||
|
||||
class NgxUrlAsyncFetcher;
|
||||
class NgxConnection;
|
||||
|
||||
class NgxConnection : public PoolElement<NgxConnection> {
|
||||
public:
|
||||
NgxConnection(MessageHandler* handler, int max_keepalive_requests);
|
||||
~NgxConnection();
|
||||
void SetSock(u_char *sockaddr, socklen_t socklen) {
|
||||
socklen_ = socklen;
|
||||
ngx_memcpy(&sockaddr_, sockaddr, socklen);
|
||||
}
|
||||
// Close ensures that NgxConnection deletes itself at the appropriate time,
|
||||
// which can be after receiving a non-keepalive response, or when the remote
|
||||
// server closes the connection when the NgxConnection is pooled and idle.
|
||||
void Close();
|
||||
|
||||
// Once keepalive is disabled, it can't be toggled back on.
|
||||
void set_keepalive(bool k) { keepalive_ = keepalive_ && k; }
|
||||
bool keepalive() { return keepalive_; }
|
||||
|
||||
typedef Pool<NgxConnection> NgxConnectionPool;
|
||||
|
||||
static NgxConnection* Connect(ngx_peer_connection_t* pc,
|
||||
MessageHandler* handler,
|
||||
int max_keepalive_requests);
|
||||
static void IdleWriteHandler(ngx_event_t* ev);
|
||||
static void IdleReadHandler(ngx_event_t* ev);
|
||||
// Terminate will cleanup any idle connections upon shutdown.
|
||||
static void Terminate();
|
||||
|
||||
static NgxConnectionPool connection_pool;
|
||||
static PthreadMutex connection_pool_mutex;
|
||||
|
||||
// c_ is owned by NgxConnection and freed in ::Close()
|
||||
ngx_connection_t* c_;
|
||||
static const int64 keepalive_timeout_ms;
|
||||
static const GoogleString ka_header;
|
||||
|
||||
private:
|
||||
int max_keepalive_requests_;
|
||||
bool keepalive_;
|
||||
socklen_t socklen_;
|
||||
u_char sockaddr_[NGX_SOCKADDRLEN];
|
||||
MessageHandler* handler_;
|
||||
|
||||
DISALLOW_COPY_AND_ASSIGN(NgxConnection);
|
||||
};
|
||||
|
||||
class NgxFetch : public PoolElement<NgxFetch> {
|
||||
public:
|
||||
NgxFetch(const GoogleString& url,
|
||||
AsyncFetch* async_fetch,
|
||||
MessageHandler* message_handler,
|
||||
ngx_log_t* log);
|
||||
~NgxFetch();
|
||||
|
||||
// Start the fetch.
|
||||
bool Start(NgxUrlAsyncFetcher* fetcher);
|
||||
// Show the completed url, for logging purposes.
|
||||
const char* str_url();
|
||||
// This fetch task is done. Call Done() on the async_fetch. It will copy the
|
||||
// buffer to cache.
|
||||
void CallbackDone(bool success);
|
||||
|
||||
// Show the bytes received.
|
||||
size_t bytes_received();
|
||||
void bytes_received_add(int64 x);
|
||||
int64 fetch_start_ms();
|
||||
void set_fetch_start_ms(int64 start_ms);
|
||||
int64 fetch_end_ms();
|
||||
void set_fetch_end_ms(int64 end_ms);
|
||||
MessageHandler* message_handler();
|
||||
|
||||
int get_major_version() {
|
||||
return static_cast<int>(status_->http_version / 1000);
|
||||
}
|
||||
int get_minor_version() {
|
||||
return static_cast<int>(status_->http_version % 1000);
|
||||
}
|
||||
int get_status_code() {
|
||||
return static_cast<int>(status_->code);
|
||||
}
|
||||
ngx_event_t* timeout_event() {
|
||||
return timeout_event_;
|
||||
}
|
||||
void set_timeout_event(ngx_event_t* x) {
|
||||
timeout_event_ = x;
|
||||
}
|
||||
void release_resolver() {
|
||||
if (resolver_ctx_ != NULL && resolver_ctx_ != NGX_NO_RESOLVER) {
|
||||
ngx_resolve_name_done(resolver_ctx_);
|
||||
resolver_ctx_ = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
response_handler_pt response_handler;
|
||||
// Do the initialized work and start the resolver work.
|
||||
bool Init();
|
||||
bool ParseUrl();
|
||||
// Prepare the request and write it to remote server.
|
||||
int InitRequest();
|
||||
// Create the connection with remote server.
|
||||
int Connect();
|
||||
void set_response_handler(response_handler_pt handler) {
|
||||
response_handler = handler;
|
||||
}
|
||||
// Only the Static functions could be used in callbacks.
|
||||
static void ResolveDoneHandler(ngx_resolver_ctx_t* ctx);
|
||||
// Write the request.
|
||||
static void ConnectionWriteHandler(ngx_event_t* wev);
|
||||
// Wait for the response.
|
||||
static void ConnectionReadHandler(ngx_event_t* rev);
|
||||
// Read and parse the first status line.
|
||||
static bool HandleStatusLine(ngx_connection_t* c);
|
||||
// Read and parse the HTTP headers.
|
||||
static bool HandleHeader(ngx_connection_t* c);
|
||||
// Read the response body.
|
||||
static bool HandleBody(ngx_connection_t* c);
|
||||
// Cancel the fetch when it's timeout.
|
||||
static void TimeoutHandler(ngx_event_t* tev);
|
||||
|
||||
// Add the pagespeed User-Agent.
|
||||
void FixUserAgent();
|
||||
void FixHost();
|
||||
|
||||
const GoogleString str_url_;
|
||||
ngx_url_t url_;
|
||||
NgxUrlAsyncFetcher* fetcher_;
|
||||
AsyncFetch* async_fetch_;
|
||||
ResponseHeadersParser parser_;
|
||||
MessageHandler* message_handler_;
|
||||
int64 bytes_received_;
|
||||
int64 fetch_start_ms_;
|
||||
int64 fetch_end_ms_;
|
||||
bool done_;
|
||||
int64 content_length_;
|
||||
bool content_length_known_;
|
||||
|
||||
struct sockaddr_in sin_;
|
||||
ngx_log_t* log_;
|
||||
ngx_buf_t* out_;
|
||||
ngx_buf_t* in_;
|
||||
ngx_pool_t* pool_;
|
||||
ngx_http_request_t* r_;
|
||||
ngx_http_status_t* status_;
|
||||
ngx_event_t* timeout_event_;
|
||||
NgxConnection* connection_;
|
||||
ngx_resolver_ctx_t* resolver_ctx_;
|
||||
|
||||
DISALLOW_COPY_AND_ASSIGN(NgxFetch);
|
||||
};
|
||||
|
||||
} // namespace net_instaweb
|
||||
|
||||
#endif // NET_INSTAWEB_NGX_FETCH_H_
|
||||
403
ngx_pagespeed-latest-beta/src/ngx_gzip_setter.cc
Normal file
403
ngx_pagespeed-latest-beta/src/ngx_gzip_setter.cc
Normal file
@@ -0,0 +1,403 @@
|
||||
/*
|
||||
* Copyright 2014 Google Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Author: kspoelstra@we-amp.com (Kees Spoelstra)
|
||||
|
||||
#include "ngx_gzip_setter.h"
|
||||
|
||||
#include <ngx_conf_file.h>
|
||||
|
||||
namespace net_instaweb {
|
||||
|
||||
NgxGZipSetter g_gzip_setter;
|
||||
|
||||
extern "C" {
|
||||
// These functions replace the setters for:
|
||||
// gzip
|
||||
// gzip_types
|
||||
// gzip_http_version
|
||||
// gzip_vary
|
||||
//
|
||||
// If these functions are called it means there is an explicit gzip
|
||||
// configuration. The gzip configuration set by pagespeed is then rolled
|
||||
// back and pagespeed will stop enabling gzip automatically.
|
||||
char* ngx_gzip_redirect_conf_set_flag_slot(
|
||||
ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
|
||||
if (g_gzip_setter.enabled()) {
|
||||
g_gzip_setter.RollBackAndDisable(cf);
|
||||
}
|
||||
char* ret = ngx_conf_set_flag_slot(cf, cmd, conf);
|
||||
return ret;
|
||||
}
|
||||
|
||||
char* ngx_gzip_redirect_http_types_slot(
|
||||
ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
|
||||
if (g_gzip_setter.enabled()) {
|
||||
g_gzip_setter.RollBackAndDisable(cf);
|
||||
}
|
||||
char* ret = ngx_http_types_slot(cf, cmd, conf);
|
||||
return ret;
|
||||
}
|
||||
|
||||
char* ngx_gzip_redirect_conf_set_enum_slot(
|
||||
ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
|
||||
if (g_gzip_setter.enabled()) {
|
||||
g_gzip_setter.RollBackAndDisable(cf);
|
||||
}
|
||||
char* ret = ngx_conf_set_enum_slot(cf, cmd, conf);
|
||||
return ret;
|
||||
}
|
||||
char* ngx_gzip_redirect_conf_set_bitmask_slot(
|
||||
ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
|
||||
if (g_gzip_setter.enabled()) {
|
||||
g_gzip_setter.RollBackAndDisable(cf);
|
||||
}
|
||||
char* ret = ngx_conf_set_bitmask_slot(cf, cmd, conf);
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
NgxGZipSetter::NgxGZipSetter() : enabled_(0) { }
|
||||
NgxGZipSetter::~NgxGZipSetter() { }
|
||||
|
||||
// Helper functions to determine signature.
|
||||
bool HasLocalConfig(ngx_command_t* command) {
|
||||
return (!(command->type & (NGX_DIRECT_CONF|NGX_MAIN_CONF)) &&
|
||||
command->conf == NGX_HTTP_LOC_CONF_OFFSET);
|
||||
}
|
||||
bool IsNgxFlagCommand(ngx_command_t* command) {
|
||||
return (command->set == ngx_conf_set_flag_slot &&
|
||||
HasLocalConfig(command));
|
||||
}
|
||||
bool IsNgxHttpTypesCommand(ngx_command_t* command) {
|
||||
return (command->set == ngx_http_types_slot &&
|
||||
HasLocalConfig(command));
|
||||
}
|
||||
bool IsNgxEnumCommand(ngx_command_t* command) {
|
||||
return (command->set == ngx_conf_set_enum_slot &&
|
||||
HasLocalConfig(command));
|
||||
}
|
||||
bool IsNgxBitmaskCommand(ngx_command_t* command) {
|
||||
return (command->set == ngx_conf_set_bitmask_slot &&
|
||||
HasLocalConfig(command));
|
||||
}
|
||||
|
||||
// Initialize the NgxGzipSetter.
|
||||
// Find the gzip, gzip_vary, gzip_http_version and gzip_types commands in the
|
||||
// gzip module. Enable if the signature of the zip command matches with what we
|
||||
// trust. Also sets up redirects for the configurations. These redirect handle
|
||||
// a rollback if expicit configuration is found.
|
||||
// If commands are not found the method will inform the user by logging.
|
||||
void NgxGZipSetter::Init(ngx_conf_t* cf) {
|
||||
#if (NGX_HTTP_GZIP)
|
||||
bool gzip_signature_mismatch = false;
|
||||
bool other_signature_mismatch = false;
|
||||
for (int m = 0; ngx_modules[m] != NULL; m++) {
|
||||
if (ngx_modules[m]->commands != NULL) {
|
||||
for (int c = 0; ngx_modules[m]->commands[c].name.len; c++) {
|
||||
ngx_command_t* current_command =& ngx_modules[m]->commands[c];
|
||||
|
||||
// We look for the gzip command, and the exact signature we trust
|
||||
// this means configured as an config location offset
|
||||
// and a ngx_flag_t setter.
|
||||
// Also see:
|
||||
// ngx_conf_handler in ngx_conf_file.c
|
||||
// ngx_http_gzip_filter_commands in ngx_http_gzip_filter.c
|
||||
if (gzip_command_.command_ == NULL &&
|
||||
STR_EQ_LITERAL(current_command->name, "gzip")) {
|
||||
if (IsNgxFlagCommand(current_command)) {
|
||||
current_command->set = ngx_gzip_redirect_conf_set_flag_slot;
|
||||
gzip_command_.command_ = current_command;
|
||||
gzip_command_.module_ = ngx_modules[m];
|
||||
enabled_ = 1;
|
||||
} else {
|
||||
ngx_conf_log_error(
|
||||
NGX_LOG_WARN, cf, 0,
|
||||
"pagespeed: cannot set gzip, signature mismatch");
|
||||
gzip_signature_mismatch = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (!gzip_http_version_command_.command_ &&
|
||||
STR_EQ_LITERAL(current_command->name, "gzip_http_version")) {
|
||||
if (IsNgxEnumCommand(current_command)) {
|
||||
current_command->set = ngx_gzip_redirect_conf_set_enum_slot;
|
||||
gzip_http_version_command_.command_ = current_command;
|
||||
gzip_http_version_command_.module_ = ngx_modules[m];
|
||||
} else {
|
||||
ngx_conf_log_error(
|
||||
NGX_LOG_WARN, cf, 0,
|
||||
"pagespeed: cannot set gzip_http_version, signature mismatch");
|
||||
other_signature_mismatch = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (!gzip_proxied_command_.command_ &&
|
||||
STR_EQ_LITERAL(current_command->name, "gzip_proxied")) {
|
||||
if (IsNgxBitmaskCommand(current_command)) {
|
||||
current_command->set = ngx_gzip_redirect_conf_set_bitmask_slot;
|
||||
gzip_proxied_command_.command_ = current_command;
|
||||
gzip_proxied_command_.module_ = ngx_modules[m];
|
||||
} else {
|
||||
ngx_conf_log_error(
|
||||
NGX_LOG_WARN, cf, 0,
|
||||
"pagespeed: cannot set gzip_proxied, signature mismatch");
|
||||
other_signature_mismatch = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (!gzip_http_types_command_.command_ &&
|
||||
STR_EQ_LITERAL(current_command->name, "gzip_types")) {
|
||||
if (IsNgxHttpTypesCommand(current_command)) {
|
||||
current_command->set = ngx_gzip_redirect_http_types_slot;
|
||||
gzip_http_types_command_.command_ = current_command;
|
||||
gzip_http_types_command_.module_ = ngx_modules[m];
|
||||
} else {
|
||||
ngx_conf_log_error(
|
||||
NGX_LOG_WARN, cf, 0,
|
||||
"pagespeed: cannot set gzip_types, signature mismatch");
|
||||
other_signature_mismatch = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (!gzip_vary_command_.command_ &&
|
||||
STR_EQ_LITERAL(current_command->name, "gzip_vary")) {
|
||||
if (IsNgxFlagCommand(current_command)) {
|
||||
current_command->set = ngx_gzip_redirect_conf_set_flag_slot;
|
||||
gzip_vary_command_.command_ = current_command;
|
||||
gzip_vary_command_.module_ = ngx_modules[m];
|
||||
} else {
|
||||
ngx_conf_log_error(
|
||||
NGX_LOG_WARN, cf, 0,
|
||||
"pagespeed: cannot set gzip_vary, signature mismatch");
|
||||
other_signature_mismatch = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (gzip_signature_mismatch) {
|
||||
return; // Already logged error.
|
||||
} else if (!enabled_) {
|
||||
// Looked through all the available commands and didn't find the "gzip" one.
|
||||
ngx_conf_log_error(
|
||||
NGX_LOG_WARN, cf, 0, "pagespeed: cannot set gzip, command not found");
|
||||
return;
|
||||
} else if (other_signature_mismatch) {
|
||||
return; // Already logged error.
|
||||
} else if (!gzip_vary_command_.command_) {
|
||||
ngx_conf_log_error(
|
||||
NGX_LOG_WARN, cf, 0, "pagespeed: missing gzip_vary");
|
||||
return;
|
||||
} else if (!gzip_http_types_command_.command_) {
|
||||
ngx_conf_log_error(
|
||||
NGX_LOG_WARN, cf, 0, "pagespeed: missing gzip_types");
|
||||
return;
|
||||
} else if (!gzip_http_version_command_.command_) {
|
||||
ngx_conf_log_error(
|
||||
NGX_LOG_WARN, cf, 0, "pagespeed: missing gzip_http_version");
|
||||
return;
|
||||
} else if (!gzip_proxied_command_.command_) {
|
||||
ngx_conf_log_error(
|
||||
NGX_LOG_WARN, cf, 0, "pagespeed: missing gzip_proxied");
|
||||
return;
|
||||
} else {
|
||||
return; // Success.
|
||||
}
|
||||
#else
|
||||
ngx_conf_log_error(
|
||||
NGX_LOG_WARN, cf, 0, "pagespeed: gzip not compiled into nginx");
|
||||
return;
|
||||
#endif
|
||||
}
|
||||
|
||||
void* ngx_command_ctx::GetConfPtr(ngx_conf_t* cf) {
|
||||
return GetModuleConfPtr(cf) + command_->offset;
|
||||
}
|
||||
|
||||
char* ngx_command_ctx::GetModuleConfPtr(ngx_conf_t* cf) {
|
||||
return reinterpret_cast<char*>(
|
||||
ngx_http_conf_get_module_loc_conf(cf, (*(module_))));
|
||||
}
|
||||
|
||||
void NgxGZipSetter::SetNgxConfFlag(ngx_conf_t* cf,
|
||||
ngx_command_ctx* command_ctx,
|
||||
ngx_flag_t value) {
|
||||
ngx_flag_t* flag = reinterpret_cast<ngx_flag_t*>(command_ctx->GetConfPtr(cf));
|
||||
*flag = value;
|
||||
// Save the flag position for possible rollback.
|
||||
ngx_flags_set_.push_back(flag);
|
||||
}
|
||||
|
||||
void NgxGZipSetter::SetNgxConfEnum(ngx_conf_t* cf,
|
||||
ngx_command_ctx* command_ctx,
|
||||
ngx_uint_t value) {
|
||||
ngx_uint_t* enum_to_set =
|
||||
reinterpret_cast<ngx_uint_t*>(command_ctx->GetConfPtr(cf));
|
||||
*enum_to_set = value;
|
||||
ngx_uint_set_.push_back(enum_to_set);
|
||||
}
|
||||
|
||||
void NgxGZipSetter::SetNgxConfBitmask(ngx_conf_t* cf,
|
||||
ngx_command_ctx* command_ctx,
|
||||
ngx_uint_t value) {
|
||||
ngx_uint_t* enum_to_set =
|
||||
reinterpret_cast<ngx_uint_t*>(command_ctx->GetConfPtr(cf));
|
||||
*enum_to_set = value;
|
||||
ngx_uint_set_.push_back(enum_to_set);
|
||||
}
|
||||
|
||||
// These are the content types we want to compress.
|
||||
ngx_str_t gzip_http_types[] = {
|
||||
ngx_string("application/ecmascript"),
|
||||
ngx_string("application/javascript"),
|
||||
ngx_string("application/json"),
|
||||
ngx_string("application/pdf"),
|
||||
ngx_string("application/postscript"),
|
||||
ngx_string("application/x-javascript"),
|
||||
ngx_string("image/svg+xml"),
|
||||
ngx_string("text/css"),
|
||||
ngx_string("text/csv"),
|
||||
// ngx_string("text/html"), // This is the default implied value.
|
||||
ngx_string("text/javascript"),
|
||||
ngx_string("text/plain"),
|
||||
ngx_string("text/xml"),
|
||||
ngx_null_string // Indicates end of array.
|
||||
};
|
||||
|
||||
gzs_enable_result NgxGZipSetter::SetGZipForLocation(ngx_conf_t* cf,
|
||||
bool value) {
|
||||
if (!enabled_) {
|
||||
return kEnableGZipNotEnabled;
|
||||
}
|
||||
if (gzip_command_.command_) {
|
||||
SetNgxConfFlag(cf, &gzip_command_, value);
|
||||
}
|
||||
return kEnableGZipOk;
|
||||
}
|
||||
|
||||
void NgxGZipSetter::EnableGZipForLocation(ngx_conf_t* cf) {
|
||||
if (!enabled_) {
|
||||
return;
|
||||
}
|
||||
|
||||
// When we get called twice for the same location{}, we ignore the second call
|
||||
// to prevent adding duplicate gzip http types and so on.
|
||||
ngx_flag_t* flag =
|
||||
reinterpret_cast<ngx_flag_t*>(gzip_command_.GetConfPtr(cf));
|
||||
if (*flag == 1) {
|
||||
return;
|
||||
}
|
||||
SetGZipForLocation(cf, true);
|
||||
if (gzip_vary_command_.command_) {
|
||||
SetNgxConfFlag(cf, &gzip_vary_command_, 1);
|
||||
}
|
||||
if (gzip_http_version_command_.command_) {
|
||||
SetNgxConfEnum(cf, &gzip_http_version_command_, NGX_HTTP_VERSION_10);
|
||||
}
|
||||
if (gzip_proxied_command_.command_) {
|
||||
SetNgxConfBitmask(
|
||||
cf, &gzip_http_version_command_, NGX_HTTP_GZIP_PROXIED_ANY);
|
||||
}
|
||||
|
||||
// This is actually the most prone to future API changes, because gzip_types
|
||||
// is not a simple type like ngx_flag_t. The signature check should be enough
|
||||
// to prevent problems.
|
||||
AddGZipHTTPTypes(cf);
|
||||
return;
|
||||
}
|
||||
|
||||
void NgxGZipSetter::AddGZipHTTPTypes(ngx_conf_t* cf) {
|
||||
if (gzip_http_types_command_.command_) {
|
||||
// Following should not happen, but if it does return gracefully.
|
||||
if (cf->args->nalloc < 2) {
|
||||
ngx_conf_log_error(NGX_LOG_WARN, cf, 0,
|
||||
"pagespeed: unexpected small cf->args in gzip_types");
|
||||
return;
|
||||
}
|
||||
|
||||
ngx_command_t* command = gzip_http_types_command_.command_;
|
||||
char* gzip_conf = reinterpret_cast<char* >(
|
||||
gzip_http_types_command_.GetModuleConfPtr(cf));
|
||||
|
||||
// Backup the old settings.
|
||||
ngx_str_t old_elt0 = reinterpret_cast<ngx_str_t*>(cf->args->elts)[0];
|
||||
ngx_str_t old_elt1 = reinterpret_cast<ngx_str_t*>(cf->args->elts)[1];
|
||||
ngx_uint_t old_nelts = cf->args->nelts;
|
||||
|
||||
// Setup first arg.
|
||||
ngx_str_t gzip_types_string = ngx_string("gzip_types");
|
||||
reinterpret_cast<ngx_str_t*>(cf->args->elts)[0] = gzip_types_string;
|
||||
cf->args->nelts = 2;
|
||||
|
||||
ngx_str_t* http_types = gzip_http_types;
|
||||
while (http_types->data) {
|
||||
ngx_str_t d;
|
||||
// We allocate the http type on the configuration pool and actually
|
||||
// leak this if we rollback. This does not seem to be a big problem,
|
||||
// because nginx also allocates tokens in ngx_conf_file.c and does not
|
||||
// free them. This way they can be used safely by configurations.
|
||||
// We must use a copy of gzip_http_types array here because nginx will
|
||||
// manipulate the values.
|
||||
// TODO(kspoelstra): better would be to allocate once on init and not
|
||||
// every time we enable gzip. This needs further investigation, sharing
|
||||
// tokens might be problematic.
|
||||
// For now I think it is not a large problem. This might add up in case
|
||||
// of a large multi server/location config with a lot of "pagespeed on"
|
||||
// directives.
|
||||
// Estimates are 300-400KB for 1000 times "pagespeed on".
|
||||
d.data = reinterpret_cast<u_char*>(
|
||||
ngx_pnalloc(cf->pool, http_types->len + 1));
|
||||
snprintf(reinterpret_cast<char*>(d.data), http_types->len + 1, "%s",
|
||||
reinterpret_cast<const char*>(http_types->data));
|
||||
d.len = http_types->len;
|
||||
reinterpret_cast<ngx_str_t*>(cf->args->elts)[1] = d;
|
||||
// Call the original setter.
|
||||
ngx_http_types_slot(cf, command, gzip_conf);
|
||||
http_types++;
|
||||
}
|
||||
|
||||
// Restore args.
|
||||
cf->args->nelts = old_nelts;
|
||||
reinterpret_cast<ngx_str_t*>(cf->args->elts)[1] = old_elt1;
|
||||
reinterpret_cast<ngx_str_t*>(cf->args->elts)[0] = old_elt0;
|
||||
|
||||
// Backup configuration location for rollback.
|
||||
ngx_httptypes_set_.push_back(gzip_conf + command->offset);
|
||||
}
|
||||
}
|
||||
|
||||
void NgxGZipSetter::RollBackAndDisable(ngx_conf_t* cf) {
|
||||
ngx_conf_log_error(NGX_LOG_INFO, cf, 0,
|
||||
"pagespeed: rollback gzip, explicit configuration");
|
||||
for (std::vector<ngx_flag_t*>::iterator i = ngx_flags_set_.begin();
|
||||
i != ngx_flags_set_.end(); ++i) {
|
||||
*(*i)=NGX_CONF_UNSET;
|
||||
}
|
||||
for (std::vector<ngx_uint_t*>::iterator i = ngx_uint_set_.begin();
|
||||
i != ngx_uint_set_.end(); ++i) {
|
||||
*(*i)=NGX_CONF_UNSET_UINT;
|
||||
}
|
||||
for (std::vector<void*>::iterator i = ngx_httptypes_set_.begin();
|
||||
i != ngx_httptypes_set_.end(); ++i) {
|
||||
ngx_array_t** type_array = reinterpret_cast<ngx_array_t**>(*i);
|
||||
ngx_array_destroy(*type_array);
|
||||
*type_array = NULL;
|
||||
}
|
||||
enabled_ = 0;
|
||||
}
|
||||
|
||||
} // namespace net_instaweb
|
||||
124
ngx_pagespeed-latest-beta/src/ngx_gzip_setter.h
Normal file
124
ngx_pagespeed-latest-beta/src/ngx_gzip_setter.h
Normal file
@@ -0,0 +1,124 @@
|
||||
/*
|
||||
* Copyright 2014 Google Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Author: kspoelstra@we-amp.com (Kees Spoelstra)
|
||||
|
||||
/*
|
||||
* NgxGZipSetter sets up gzip for pagespeed
|
||||
* with the following configuration:
|
||||
* gzip on;
|
||||
* gzip_vary on;
|
||||
* gzip_types application/ecmascript;
|
||||
* gzip_types application/javascript;
|
||||
* gzip_types application/json;
|
||||
* gzip_types application/pdf;
|
||||
* gzip_types application/postscript;
|
||||
* gzip_types application/x-javascript;
|
||||
* gzip_types image/svg+xml;
|
||||
* gzip_types text/css;
|
||||
* gzip_types text/csv;
|
||||
* gzip_types text/javascript;
|
||||
* gzip_types text/plain;
|
||||
* gzip_types text/xml;
|
||||
* gzip_http_version 1.0;
|
||||
*
|
||||
* If there is an explicit gzip configuration in the nginx.conf
|
||||
* pagespeed will rollback the set configuration and let the
|
||||
* user decide what the configuration will be.
|
||||
*
|
||||
* It manipulates the configuration by manipulating ngx_flag_t
|
||||
* and ngx_uint_t settings directly and using the nginx setter for
|
||||
* gzip_http_types.
|
||||
* This is probably a safe way to do it. If this mechanism
|
||||
* changes all non nginx module setup & configuration will
|
||||
* fail.
|
||||
*/
|
||||
|
||||
#ifndef NGX_GZIP_SETTER_H_
|
||||
#define NGX_GZIP_SETTER_H_
|
||||
|
||||
extern "C" {
|
||||
#include <ngx_config.h>
|
||||
#include <ngx_core.h>
|
||||
#include <ngx_http.h>
|
||||
}
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "ngx_pagespeed.h"
|
||||
|
||||
#include "pagespeed/kernel/base/basictypes.h"
|
||||
|
||||
namespace net_instaweb {
|
||||
|
||||
// We need this class because configuration for gzip is in different modules, so
|
||||
// just saving the command will not work.
|
||||
class ngx_command_ctx {
|
||||
public:
|
||||
ngx_command_ctx() : command_(NULL), module_(NULL) { }
|
||||
void* GetConfPtr(ngx_conf_t* cf);
|
||||
char* GetModuleConfPtr(ngx_conf_t* cf);
|
||||
ngx_command_t* command_;
|
||||
ngx_module_t* module_;
|
||||
};
|
||||
|
||||
enum gzs_enable_result {
|
||||
kEnableGZipOk,
|
||||
kEnableGZipPartial,
|
||||
kEnableGZipNotEnabled
|
||||
};
|
||||
|
||||
class NgxGZipSetter {
|
||||
std::vector<ngx_flag_t*> ngx_flags_set_;
|
||||
std::vector<ngx_uint_t*> ngx_uint_set_;
|
||||
std::vector<void*> ngx_httptypes_set_;
|
||||
ngx_command_ctx gzip_command_;
|
||||
ngx_command_ctx gzip_http_types_command_;
|
||||
ngx_command_ctx gzip_proxied_command_;
|
||||
ngx_command_ctx gzip_vary_command_;
|
||||
ngx_command_ctx gzip_http_version_command_;
|
||||
bool enabled_;
|
||||
|
||||
public:
|
||||
NgxGZipSetter();
|
||||
~NgxGZipSetter();
|
||||
void Init(ngx_conf_t* cf);
|
||||
|
||||
void SetNgxConfFlag(ngx_conf_t* cf,
|
||||
ngx_command_ctx* command_ctx,
|
||||
ngx_flag_t value);
|
||||
void SetNgxConfEnum(ngx_conf_t* cf,
|
||||
ngx_command_ctx* command_ctx,
|
||||
ngx_uint_t value);
|
||||
void SetNgxConfBitmask(ngx_conf_t* cf,
|
||||
ngx_command_ctx* command_ctx,
|
||||
ngx_uint_t value);
|
||||
void EnableGZipForLocation(ngx_conf_t* cf);
|
||||
gzs_enable_result SetGZipForLocation(ngx_conf_t* cf, bool value);
|
||||
void AddGZipHTTPTypes(ngx_conf_t* cf);
|
||||
void RollBackAndDisable(ngx_conf_t* cf);
|
||||
|
||||
bool enabled() { return enabled_; }
|
||||
|
||||
private:
|
||||
DISALLOW_COPY_AND_ASSIGN(NgxGZipSetter);
|
||||
};
|
||||
|
||||
extern NgxGZipSetter g_gzip_setter;
|
||||
|
||||
} // namespace net_instaweb
|
||||
|
||||
#endif // NGX_GZIP_SETTER_H_
|
||||
39
ngx_pagespeed-latest-beta/src/ngx_list_iterator.cc
Normal file
39
ngx_pagespeed-latest-beta/src/ngx_list_iterator.cc
Normal file
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
* Copyright 2013 Google Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Author: jefftk@google.com (Jeff Kaufman)
|
||||
|
||||
#include "ngx_list_iterator.h"
|
||||
|
||||
namespace net_instaweb {
|
||||
|
||||
NgxListIterator::NgxListIterator(ngx_list_part_t* part) :
|
||||
part_(part),
|
||||
index_within_part_(0) {}
|
||||
|
||||
ngx_table_elt_t* NgxListIterator::Next() {
|
||||
if (index_within_part_ >= part_->nelts) {
|
||||
if (part_->next == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
part_ = part_->next;
|
||||
index_within_part_ = 0;
|
||||
}
|
||||
ngx_table_elt_t* elts = static_cast<ngx_table_elt_t*>(part_->elts);
|
||||
return &elts[index_within_part_++]; // Intentional post-increment.
|
||||
}
|
||||
|
||||
} // namespace net_instaweb
|
||||
50
ngx_pagespeed-latest-beta/src/ngx_list_iterator.h
Normal file
50
ngx_pagespeed-latest-beta/src/ngx_list_iterator.h
Normal file
@@ -0,0 +1,50 @@
|
||||
/*
|
||||
* Copyright 2013 Google Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Author: jefftk@google.com (Jeff Kaufman)
|
||||
//
|
||||
// Simplifies iteration over nginx lists.
|
||||
//
|
||||
//
|
||||
|
||||
#ifndef NGX_LIST_ITERATOR_H_
|
||||
#define NGX_LIST_ITERATOR_H_
|
||||
|
||||
extern "C" {
|
||||
#include <ngx_http.h>
|
||||
}
|
||||
|
||||
#include "base/basictypes.h"
|
||||
|
||||
namespace net_instaweb {
|
||||
|
||||
class NgxListIterator {
|
||||
public:
|
||||
explicit NgxListIterator(ngx_list_part_t* part);
|
||||
|
||||
// Return the next element of the list if there is one, NULL otherwise.
|
||||
ngx_table_elt_t* Next();
|
||||
|
||||
private:
|
||||
ngx_list_part_t* part_;
|
||||
ngx_uint_t index_within_part_;
|
||||
|
||||
DISALLOW_COPY_AND_ASSIGN(NgxListIterator);
|
||||
};
|
||||
|
||||
} // namespace net_instaweb
|
||||
|
||||
#endif // NGX_LIST_ITERATOR_H_
|
||||
116
ngx_pagespeed-latest-beta/src/ngx_message_handler.cc
Normal file
116
ngx_pagespeed-latest-beta/src/ngx_message_handler.cc
Normal file
@@ -0,0 +1,116 @@
|
||||
// Copyright 2013 Google Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Author: oschaaf@gmail.com (Otto van der Schaaf)
|
||||
|
||||
#include "ngx_message_handler.h"
|
||||
|
||||
#include <signal.h>
|
||||
|
||||
#include "net/instaweb/public/version.h"
|
||||
#include "pagespeed/kernel/base/abstract_mutex.h"
|
||||
#include "pagespeed/kernel/base/debug.h"
|
||||
#include "pagespeed/kernel/base/posix_timer.h"
|
||||
#include "pagespeed/kernel/base/string_util.h"
|
||||
#include "pagespeed/kernel/base/time_util.h"
|
||||
#include "pagespeed/kernel/sharedmem/shared_circular_buffer.h"
|
||||
|
||||
namespace {
|
||||
|
||||
// This will be prefixed to every logged message.
|
||||
const char kModuleName[] = "ngx_pagespeed";
|
||||
|
||||
// If set, the crash handler will use this to output a backtrace using
|
||||
// ngx_log_error.
|
||||
ngx_log_t* global_log = NULL;
|
||||
|
||||
} // namespace
|
||||
|
||||
extern "C" {
|
||||
static void signal_handler(int sig) {
|
||||
// Try to output the backtrace to the log file. Since this may end up
|
||||
// crashing/deadlocking/etc. we set an alarm() to abort us if it comes to
|
||||
// that.
|
||||
alarm(2);
|
||||
if (global_log != NULL) {
|
||||
ngx_log_error(NGX_LOG_ALERT, global_log, 0, "Trapped signal [%d]\n%s",
|
||||
sig, net_instaweb::StackTraceString().c_str());
|
||||
} else {
|
||||
fprintf(stderr, "Trapped signal [%d]\n%s\n",
|
||||
sig, net_instaweb::StackTraceString().c_str());
|
||||
}
|
||||
kill(getpid(), SIGKILL);
|
||||
}
|
||||
} // extern "C"
|
||||
|
||||
namespace net_instaweb {
|
||||
|
||||
NgxMessageHandler::NgxMessageHandler(Timer* timer, AbstractMutex* mutex)
|
||||
: SystemMessageHandler(timer, mutex),
|
||||
log_(NULL) {
|
||||
}
|
||||
|
||||
// Installs a signal handler for common crash signals, that tries to print
|
||||
// out a backtrace.
|
||||
void NgxMessageHandler::InstallCrashHandler(ngx_log_t* log) {
|
||||
global_log = log;
|
||||
signal(SIGTRAP, signal_handler); // On check failures
|
||||
signal(SIGABRT, signal_handler);
|
||||
signal(SIGFPE, signal_handler);
|
||||
signal(SIGSEGV, signal_handler);
|
||||
}
|
||||
|
||||
ngx_uint_t NgxMessageHandler::GetNgxLogLevel(MessageType type) {
|
||||
switch (type) {
|
||||
case kInfo:
|
||||
return NGX_LOG_INFO;
|
||||
case kWarning:
|
||||
return NGX_LOG_WARN;
|
||||
case kError:
|
||||
return NGX_LOG_ERR;
|
||||
case kFatal:
|
||||
return NGX_LOG_ALERT;
|
||||
}
|
||||
|
||||
// This should never fall through, but some compilers seem to complain if
|
||||
// we don't include this.
|
||||
return NGX_LOG_ALERT;
|
||||
}
|
||||
|
||||
void NgxMessageHandler::MessageSImpl(MessageType type,
|
||||
const GoogleString& message) {
|
||||
if (log_ != NULL) {
|
||||
ngx_uint_t log_level = GetNgxLogLevel(type);
|
||||
ngx_log_error(log_level, log_, 0/*ngx_err_t*/, "[%s %s] %s",
|
||||
kModuleName, kModPagespeedVersion, message.c_str());
|
||||
} else {
|
||||
GoogleMessageHandler::MessageSImpl(type, message);
|
||||
}
|
||||
AddMessageToBuffer(type, message);
|
||||
}
|
||||
|
||||
void NgxMessageHandler::FileMessageSImpl(
|
||||
MessageType type, const char* file, int line, const GoogleString& message) {
|
||||
if (log_ != NULL) {
|
||||
ngx_uint_t log_level = GetNgxLogLevel(type);
|
||||
ngx_log_error(log_level, log_, 0/*ngx_err_t*/, "[%s %s] %s:%d:%s",
|
||||
kModuleName, kModPagespeedVersion, file, line,
|
||||
message.c_str());
|
||||
} else {
|
||||
GoogleMessageHandler::FileMessageSImpl(type, file, line, message);
|
||||
}
|
||||
AddMessageToBuffer(type, file, line, message);
|
||||
}
|
||||
|
||||
} // namespace net_instaweb
|
||||
70
ngx_pagespeed-latest-beta/src/ngx_message_handler.h
Normal file
70
ngx_pagespeed-latest-beta/src/ngx_message_handler.h
Normal file
@@ -0,0 +1,70 @@
|
||||
// Copyright 2013 Google Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Author: oschaaf@gmail.com (Otto van der Schaaf)
|
||||
|
||||
#ifndef NGX_MESSAGE_HANDLER_H_
|
||||
#define NGX_MESSAGE_HANDLER_H_
|
||||
|
||||
extern "C" {
|
||||
#include <ngx_auto_config.h>
|
||||
#if (NGX_THREADS)
|
||||
#include <ngx_thread.h>
|
||||
#endif
|
||||
#include <ngx_core.h>
|
||||
#include <ngx_log.h>
|
||||
}
|
||||
|
||||
#include <cstdarg>
|
||||
|
||||
#include "pagespeed/kernel/base/basictypes.h"
|
||||
#include "pagespeed/kernel/base/message_handler.h"
|
||||
#include "pagespeed/kernel/base/string.h"
|
||||
#include "pagespeed/kernel/base/string_util.h"
|
||||
#include "pagespeed/system/system_message_handler.h"
|
||||
|
||||
namespace net_instaweb {
|
||||
|
||||
class AbstractMutex;
|
||||
class Timer;
|
||||
|
||||
// Implementation of a message handler that uses ngx_log_error()
|
||||
// logging to emit messages, with a fallback to GoogleMessageHandler
|
||||
class NgxMessageHandler : public SystemMessageHandler {
|
||||
public:
|
||||
explicit NgxMessageHandler(Timer* timer, AbstractMutex* mutex);
|
||||
|
||||
// Installs a signal handler for common crash signals that tries to print
|
||||
// out a backtrace.
|
||||
static void InstallCrashHandler(ngx_log_t* log);
|
||||
|
||||
void set_log(ngx_log_t* log) { log_ = log; }
|
||||
ngx_log_t* log() { return log_; }
|
||||
|
||||
protected:
|
||||
virtual void MessageSImpl(MessageType type, const GoogleString& message);
|
||||
|
||||
virtual void FileMessageSImpl(MessageType type, const char* file,
|
||||
int line, const GoogleString& message);
|
||||
|
||||
private:
|
||||
ngx_uint_t GetNgxLogLevel(MessageType type);
|
||||
ngx_log_t* log_;
|
||||
|
||||
DISALLOW_COPY_AND_ASSIGN(NgxMessageHandler);
|
||||
};
|
||||
|
||||
} // namespace net_instaweb
|
||||
|
||||
#endif // NGX_MESSAGE_HANDLER_H_
|
||||
3266
ngx_pagespeed-latest-beta/src/ngx_pagespeed.cc
Normal file
3266
ngx_pagespeed-latest-beta/src/ngx_pagespeed.cc
Normal file
File diff suppressed because it is too large
Load Diff
138
ngx_pagespeed-latest-beta/src/ngx_pagespeed.h
Normal file
138
ngx_pagespeed-latest-beta/src/ngx_pagespeed.h
Normal file
@@ -0,0 +1,138 @@
|
||||
/*
|
||||
* Copyright 2012 Google Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Author: jefftk@google.com (Jeff Kaufman)
|
||||
|
||||
#ifndef NGX_PAGESPEED_H_
|
||||
#define NGX_PAGESPEED_H_
|
||||
|
||||
// We might be compiled with syslog.h, which #defines LOG_INFO and LOG_WARNING
|
||||
// as ints. But logging.h assumes they're usable as names, within their
|
||||
// namespace, so we need to #undef them before including logging.h
|
||||
#ifdef LOG_INFO
|
||||
#undef LOG_INFO
|
||||
#endif
|
||||
#ifdef LOG_WARNING
|
||||
#undef LOG_WARNING
|
||||
#endif
|
||||
|
||||
extern "C" {
|
||||
#include <ngx_http.h>
|
||||
}
|
||||
|
||||
#include "base/logging.h"
|
||||
#include "pagespeed/kernel/base/string_util.h"
|
||||
#include "pagespeed/kernel/http/response_headers.h"
|
||||
|
||||
namespace net_instaweb {
|
||||
|
||||
class GzipInflater;
|
||||
class NgxBaseFetch;
|
||||
class ProxyFetch;
|
||||
class RewriteDriver;
|
||||
class RequestHeaders;
|
||||
class ResponseHeaders;
|
||||
class InPlaceResourceRecorder;
|
||||
|
||||
// Allocate chain links and buffers from the supplied pool, and copy over the
|
||||
// data from the string piece. If the string piece is empty, return
|
||||
// NGX_DECLINED immediately unless send_last_buf.
|
||||
ngx_int_t string_piece_to_buffer_chain(
|
||||
ngx_pool_t* pool, StringPiece sp,
|
||||
ngx_chain_t** link_ptr, bool send_last_buf, bool send_flush);
|
||||
|
||||
StringPiece str_to_string_piece(ngx_str_t s);
|
||||
|
||||
// s1: ngx_str_t, s2: string literal
|
||||
// true if they're equal, false otherwise
|
||||
#define STR_EQ_LITERAL(s1, s2) \
|
||||
((s1).len == (sizeof(s2)-1) && \
|
||||
ngx_strncmp((s1).data, (s2), (sizeof(s2)-1)) == 0)
|
||||
|
||||
// s1: ngx_str_t, s2: string literal
|
||||
// true if they're equal ignoring case, false otherwise
|
||||
#define STR_CASE_EQ_LITERAL(s1, s2) \
|
||||
((s1).len == (sizeof(s2)-1) && \
|
||||
ngx_strncasecmp((s1).data, ( \
|
||||
reinterpret_cast<u_char*>( \
|
||||
const_cast<char*>(s2))), \
|
||||
(sizeof(s2)-1)) == 0)
|
||||
|
||||
// Allocate memory out of the pool for the string piece, and copy the contents
|
||||
// over. Returns NULL if we can't get memory.
|
||||
char* string_piece_to_pool_string(ngx_pool_t* pool, StringPiece sp);
|
||||
|
||||
enum PreserveCachingHeaders {
|
||||
kPreserveAllCachingHeaders, // Cache-Control, ETag, Last-Modified, etc
|
||||
kPreserveOnlyCacheControl, // Only Cache-Control.
|
||||
kDontPreserveHeaders,
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
NgxBaseFetch* base_fetch;
|
||||
|
||||
ngx_http_request_t* r;
|
||||
|
||||
bool html_rewrite;
|
||||
bool in_place;
|
||||
|
||||
PreserveCachingHeaders preserve_caching_headers;
|
||||
|
||||
// for html rewrite
|
||||
ProxyFetch* proxy_fetch;
|
||||
GzipInflater* inflater_;
|
||||
|
||||
// for in place resource
|
||||
RewriteDriver* driver;
|
||||
InPlaceResourceRecorder* recorder;
|
||||
ResponseHeaders* ipro_response_headers;
|
||||
|
||||
// We need to remember the URL here as well since we may modify what NGX
|
||||
// gets by stripping our special query params and honoring X-Forwarded-Proto.
|
||||
GoogleString url_string;
|
||||
|
||||
// We need to remember if the upstream had headers_out->location set, because
|
||||
// we should mirror that when we write it back. nginx may absolutify
|
||||
// Location: headers that start with '/' without regarding X-Forwarded-Proto.
|
||||
bool location_field_set;
|
||||
bool psol_vary_accept_only;
|
||||
bool follow_flushes;
|
||||
} ps_request_ctx_t;
|
||||
|
||||
ps_request_ctx_t* ps_get_request_context(ngx_http_request_t* r);
|
||||
|
||||
void copy_request_headers_from_ngx(const ngx_http_request_t* r,
|
||||
RequestHeaders* headers);
|
||||
|
||||
void copy_response_headers_from_ngx(const ngx_http_request_t* r,
|
||||
ResponseHeaders* headers);
|
||||
|
||||
ngx_int_t copy_response_headers_to_ngx(
|
||||
ngx_http_request_t* r,
|
||||
const ResponseHeaders& pagespeed_headers,
|
||||
PreserveCachingHeaders preserve_caching_headers);
|
||||
|
||||
StringPiece ps_determine_host(ngx_http_request_t* r);
|
||||
|
||||
namespace ps_base_fetch {
|
||||
|
||||
ngx_int_t ps_base_fetch_handler(ngx_http_request_t* r);
|
||||
|
||||
} // namespace ps_base_fetch
|
||||
|
||||
} // namespace net_instaweb
|
||||
|
||||
#endif // NGX_PAGESPEED_H_
|
||||
288
ngx_pagespeed-latest-beta/src/ngx_rewrite_driver_factory.cc
Normal file
288
ngx_pagespeed-latest-beta/src/ngx_rewrite_driver_factory.cc
Normal file
@@ -0,0 +1,288 @@
|
||||
/*
|
||||
* Copyright 2012 Google Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Author: jefftk@google.com (Jeff Kaufman)
|
||||
|
||||
#include "ngx_rewrite_driver_factory.h"
|
||||
|
||||
#include <cstdio>
|
||||
|
||||
#include "log_message_handler.h"
|
||||
#include "ngx_message_handler.h"
|
||||
#include "ngx_rewrite_options.h"
|
||||
#include "ngx_server_context.h"
|
||||
#include "ngx_url_async_fetcher.h"
|
||||
|
||||
#include "net/instaweb/http/public/rate_controller.h"
|
||||
#include "net/instaweb/http/public/rate_controlling_url_async_fetcher.h"
|
||||
#include "net/instaweb/http/public/wget_url_fetcher.h"
|
||||
#include "net/instaweb/rewriter/public/rewrite_driver.h"
|
||||
#include "net/instaweb/rewriter/public/rewrite_driver_factory.h"
|
||||
#include "net/instaweb/rewriter/public/server_context.h"
|
||||
#include "net/instaweb/util/public/property_cache.h"
|
||||
#include "pagespeed/kernel/base/google_message_handler.h"
|
||||
#include "pagespeed/kernel/base/null_shared_mem.h"
|
||||
#include "pagespeed/kernel/base/posix_timer.h"
|
||||
#include "pagespeed/kernel/base/stdio_file_system.h"
|
||||
#include "pagespeed/kernel/base/string.h"
|
||||
#include "pagespeed/kernel/base/string_util.h"
|
||||
#include "pagespeed/kernel/base/thread_system.h"
|
||||
#include "pagespeed/kernel/http/content_type.h"
|
||||
#include "pagespeed/kernel/sharedmem/shared_circular_buffer.h"
|
||||
#include "pagespeed/kernel/sharedmem/shared_mem_statistics.h"
|
||||
#include "pagespeed/kernel/thread/pthread_shared_mem.h"
|
||||
#include "pagespeed/kernel/thread/scheduler_thread.h"
|
||||
#include "pagespeed/kernel/thread/slow_worker.h"
|
||||
#include "pagespeed/system/in_place_resource_recorder.h"
|
||||
#include "pagespeed/system/serf_url_async_fetcher.h"
|
||||
#include "pagespeed/system/system_caches.h"
|
||||
#include "pagespeed/system/system_rewrite_options.h"
|
||||
|
||||
namespace net_instaweb {
|
||||
|
||||
class FileSystem;
|
||||
class Hasher;
|
||||
class MessageHandler;
|
||||
class Statistics;
|
||||
class Timer;
|
||||
class UrlAsyncFetcher;
|
||||
class UrlFetcher;
|
||||
class Writer;
|
||||
|
||||
class SharedCircularBuffer;
|
||||
|
||||
NgxRewriteDriverFactory::NgxRewriteDriverFactory(
|
||||
const ProcessContext& process_context,
|
||||
SystemThreadSystem* system_thread_system, StringPiece hostname, int port)
|
||||
: SystemRewriteDriverFactory(process_context, system_thread_system,
|
||||
NULL /* default shared memory runtime */, hostname, port),
|
||||
threads_started_(false),
|
||||
ngx_message_handler_(
|
||||
new NgxMessageHandler(timer(), thread_system()->NewMutex())),
|
||||
ngx_html_parse_message_handler_(
|
||||
new NgxMessageHandler(timer(), thread_system()->NewMutex())),
|
||||
log_(NULL),
|
||||
resolver_timeout_(NGX_CONF_UNSET_MSEC),
|
||||
use_native_fetcher_(false),
|
||||
// 100 Aligns to nginx's server-side default.
|
||||
native_fetcher_max_keepalive_requests_(100),
|
||||
ngx_shared_circular_buffer_(NULL),
|
||||
hostname_(hostname.as_string()),
|
||||
port_(port),
|
||||
process_script_variables_mode_(ProcessScriptVariablesMode::kOff),
|
||||
process_script_variables_set_(false),
|
||||
shut_down_(false) {
|
||||
InitializeDefaultOptions();
|
||||
default_options()->set_beacon_url("/ngx_pagespeed_beacon");
|
||||
SystemRewriteOptions* system_options = dynamic_cast<SystemRewriteOptions*>(
|
||||
default_options());
|
||||
system_options->set_file_cache_clean_inode_limit(500000);
|
||||
system_options->set_avoid_renaming_introspective_javascript(true);
|
||||
set_message_handler(ngx_message_handler_);
|
||||
set_html_parse_message_handler(ngx_html_parse_message_handler_);
|
||||
}
|
||||
|
||||
NgxRewriteDriverFactory::~NgxRewriteDriverFactory() {
|
||||
ShutDown();
|
||||
ngx_shared_circular_buffer_ = NULL;
|
||||
STLDeleteElements(&uninitialized_server_contexts_);
|
||||
}
|
||||
|
||||
Hasher* NgxRewriteDriverFactory::NewHasher() {
|
||||
return new MD5Hasher;
|
||||
}
|
||||
|
||||
UrlAsyncFetcher* NgxRewriteDriverFactory::AllocateFetcher(
|
||||
SystemRewriteOptions* config) {
|
||||
if (use_native_fetcher_) {
|
||||
NgxUrlAsyncFetcher* fetcher = new NgxUrlAsyncFetcher(
|
||||
config->fetcher_proxy().c_str(),
|
||||
log_,
|
||||
resolver_timeout_,
|
||||
config->blocking_fetch_timeout_ms(),
|
||||
resolver_,
|
||||
native_fetcher_max_keepalive_requests_,
|
||||
thread_system(),
|
||||
message_handler());
|
||||
ngx_url_async_fetchers_.push_back(fetcher);
|
||||
return fetcher;
|
||||
} else {
|
||||
return SystemRewriteDriverFactory::AllocateFetcher(config);
|
||||
}
|
||||
}
|
||||
|
||||
MessageHandler* NgxRewriteDriverFactory::DefaultHtmlParseMessageHandler() {
|
||||
return ngx_html_parse_message_handler_;
|
||||
}
|
||||
|
||||
MessageHandler* NgxRewriteDriverFactory::DefaultMessageHandler() {
|
||||
return ngx_message_handler_;
|
||||
}
|
||||
|
||||
FileSystem* NgxRewriteDriverFactory::DefaultFileSystem() {
|
||||
return new StdioFileSystem();
|
||||
}
|
||||
|
||||
Timer* NgxRewriteDriverFactory::DefaultTimer() {
|
||||
return new PosixTimer;
|
||||
}
|
||||
|
||||
NamedLockManager* NgxRewriteDriverFactory::DefaultLockManager() {
|
||||
CHECK(false);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
RewriteOptions* NgxRewriteDriverFactory::NewRewriteOptions() {
|
||||
NgxRewriteOptions* options = new NgxRewriteOptions(thread_system());
|
||||
// TODO(jefftk): figure out why using SetDefaultRewriteLevel like
|
||||
// mod_pagespeed does in mod_instaweb.cc:create_dir_config() isn't enough here
|
||||
// -- if you use that instead then ngx_pagespeed doesn't actually end up
|
||||
// defaulting CoreFilters.
|
||||
// See: https://github.com/pagespeed/ngx_pagespeed/issues/1190
|
||||
options->SetRewriteLevel(RewriteOptions::kCoreFilters);
|
||||
return options;
|
||||
}
|
||||
|
||||
RewriteOptions* NgxRewriteDriverFactory::NewRewriteOptionsForQuery() {
|
||||
return new NgxRewriteOptions(thread_system());
|
||||
}
|
||||
|
||||
bool NgxRewriteDriverFactory::CheckResolver() {
|
||||
if (use_native_fetcher_ && resolver_ == NULL) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
NgxServerContext* NgxRewriteDriverFactory::MakeNgxServerContext(
|
||||
StringPiece hostname, int port) {
|
||||
NgxServerContext* server_context = new NgxServerContext(this, hostname, port);
|
||||
uninitialized_server_contexts_.insert(server_context);
|
||||
return server_context;
|
||||
}
|
||||
|
||||
ServerContext* NgxRewriteDriverFactory::NewDecodingServerContext() {
|
||||
ServerContext* sc = new NgxServerContext(this, hostname_, port_);
|
||||
InitStubDecodingServerContext(sc);
|
||||
return sc;
|
||||
}
|
||||
|
||||
ServerContext* NgxRewriteDriverFactory::NewServerContext() {
|
||||
LOG(DFATAL) << "MakeNgxServerContext should be used instead";
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void NgxRewriteDriverFactory::ShutDown() {
|
||||
if (!shut_down_) {
|
||||
shut_down_ = true;
|
||||
SystemRewriteDriverFactory::ShutDown();
|
||||
}
|
||||
}
|
||||
|
||||
void NgxRewriteDriverFactory::ShutDownMessageHandlers() {
|
||||
ngx_message_handler_->set_buffer(NULL);
|
||||
ngx_html_parse_message_handler_->set_buffer(NULL);
|
||||
for (NgxMessageHandlerSet::iterator p =
|
||||
server_context_message_handlers_.begin();
|
||||
p != server_context_message_handlers_.end(); ++p) {
|
||||
(*p)->set_buffer(NULL);
|
||||
}
|
||||
server_context_message_handlers_.clear();
|
||||
}
|
||||
|
||||
void NgxRewriteDriverFactory::StartThreads() {
|
||||
if (threads_started_) {
|
||||
return;
|
||||
}
|
||||
// TODO(jefftk): use a native nginx timer instead of running our own thread.
|
||||
// See issue #111.
|
||||
SchedulerThread* thread = new SchedulerThread(thread_system(), scheduler());
|
||||
bool ok = thread->Start();
|
||||
CHECK(ok) << "Unable to start scheduler thread";
|
||||
defer_cleanup(thread->MakeDeleter());
|
||||
threads_started_ = true;
|
||||
}
|
||||
|
||||
void NgxRewriteDriverFactory::SetMainConf(NgxRewriteOptions* main_options) {
|
||||
// Propagate process-scope options from the copy we had during nginx option
|
||||
// parsing to our own.
|
||||
if (main_options != NULL) {
|
||||
default_options()->MergeOnlyProcessScopeOptions(*main_options);
|
||||
}
|
||||
}
|
||||
|
||||
void NgxRewriteDriverFactory::LoggingInit(
|
||||
ngx_log_t* log, bool may_install_crash_handler) {
|
||||
log_ = log;
|
||||
net_instaweb::log_message_handler::Install(log);
|
||||
if (may_install_crash_handler && install_crash_handler()) {
|
||||
NgxMessageHandler::InstallCrashHandler(log);
|
||||
}
|
||||
ngx_message_handler_->set_log(log);
|
||||
ngx_html_parse_message_handler_->set_log(log);
|
||||
}
|
||||
|
||||
void NgxRewriteDriverFactory::SetCircularBuffer(
|
||||
SharedCircularBuffer* buffer) {
|
||||
ngx_shared_circular_buffer_ = buffer;
|
||||
ngx_message_handler_->set_buffer(buffer);
|
||||
ngx_html_parse_message_handler_->set_buffer(buffer);
|
||||
}
|
||||
|
||||
void NgxRewriteDriverFactory::SetServerContextMessageHandler(
|
||||
ServerContext* server_context, ngx_log_t* log) {
|
||||
NgxMessageHandler* handler = new NgxMessageHandler(
|
||||
timer(), thread_system()->NewMutex());
|
||||
handler->set_log(log);
|
||||
// The ngx_shared_circular_buffer_ will be NULL if MessageBufferSize hasn't
|
||||
// been raised from its default of 0.
|
||||
handler->set_buffer(ngx_shared_circular_buffer_);
|
||||
server_context_message_handlers_.insert(handler);
|
||||
defer_cleanup(new Deleter<NgxMessageHandler>(handler));
|
||||
server_context->set_message_handler(handler);
|
||||
}
|
||||
|
||||
void NgxRewriteDriverFactory::InitStats(Statistics* statistics) {
|
||||
// Init standard PSOL stats.
|
||||
SystemRewriteDriverFactory::InitStats(statistics);
|
||||
RewriteDriverFactory::InitStats(statistics);
|
||||
RateController::InitStats(statistics);
|
||||
|
||||
// Init Ngx-specific stats.
|
||||
NgxServerContext::InitStats(statistics);
|
||||
InPlaceResourceRecorder::InitStats(statistics);
|
||||
}
|
||||
|
||||
void NgxRewriteDriverFactory::PrepareForkedProcess(const char* name) {
|
||||
ngx_pid = ngx_getpid(); // Needed for logging to have the right PIDs.
|
||||
SystemRewriteDriverFactory::PrepareForkedProcess(name);
|
||||
}
|
||||
|
||||
void NgxRewriteDriverFactory::NameProcess(const char* name) {
|
||||
SystemRewriteDriverFactory::NameProcess(name);
|
||||
|
||||
// Superclass set status with prctl. Nginx has a helper function for setting
|
||||
// argv[0] as well, so let's use that. We'll show up as:
|
||||
//
|
||||
// nginx: pagespeed $name
|
||||
|
||||
char name_for_setproctitle[32];
|
||||
snprintf(name_for_setproctitle, sizeof(name_for_setproctitle),
|
||||
"pagespeed %s", name);
|
||||
ngx_setproctitle(name_for_setproctitle);
|
||||
}
|
||||
|
||||
} // namespace net_instaweb
|
||||
176
ngx_pagespeed-latest-beta/src/ngx_rewrite_driver_factory.h
Normal file
176
ngx_pagespeed-latest-beta/src/ngx_rewrite_driver_factory.h
Normal file
@@ -0,0 +1,176 @@
|
||||
/*
|
||||
* Copyright 2012 Google Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Author: jefftk@google.com (Jeff Kaufman)
|
||||
|
||||
#ifndef NGX_REWRITE_DRIVER_FACTORY_H_
|
||||
#define NGX_REWRITE_DRIVER_FACTORY_H_
|
||||
|
||||
extern "C" {
|
||||
#include <ngx_auto_config.h>
|
||||
#if (NGX_THREADS)
|
||||
#include <ngx_thread.h>
|
||||
#endif
|
||||
#include <ngx_core.h>
|
||||
#include <ngx_http.h>
|
||||
#include <ngx_config.h>
|
||||
#include <ngx_log.h>
|
||||
}
|
||||
|
||||
#include <set>
|
||||
|
||||
#include "pagespeed/kernel/base/md5_hasher.h"
|
||||
#include "pagespeed/kernel/base/scoped_ptr.h"
|
||||
#include "pagespeed/system/system_rewrite_driver_factory.h"
|
||||
|
||||
namespace net_instaweb {
|
||||
|
||||
class NgxMessageHandler;
|
||||
class NgxRewriteOptions;
|
||||
class NgxServerContext;
|
||||
class NgxUrlAsyncFetcher;
|
||||
class SharedCircularBuffer;
|
||||
class SharedMemRefererStatistics;
|
||||
class SlowWorker;
|
||||
class Statistics;
|
||||
class SystemThreadSystem;
|
||||
|
||||
enum ProcessScriptVariablesMode {
|
||||
kOff,
|
||||
kLegacyRestricted,
|
||||
kAll
|
||||
};
|
||||
|
||||
class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
|
||||
public:
|
||||
// We take ownership of the thread system.
|
||||
explicit NgxRewriteDriverFactory(
|
||||
const ProcessContext& process_context,
|
||||
SystemThreadSystem* system_thread_system, StringPiece hostname, int port);
|
||||
virtual ~NgxRewriteDriverFactory();
|
||||
virtual Hasher* NewHasher();
|
||||
virtual UrlAsyncFetcher* AllocateFetcher(SystemRewriteOptions* config);
|
||||
virtual MessageHandler* DefaultHtmlParseMessageHandler();
|
||||
virtual MessageHandler* DefaultMessageHandler();
|
||||
virtual FileSystem* DefaultFileSystem();
|
||||
virtual Timer* DefaultTimer();
|
||||
virtual NamedLockManager* DefaultLockManager();
|
||||
// Create a new RewriteOptions. In this implementation it will be an
|
||||
// NgxRewriteOptions, and it will have CoreFilters explicitly set.
|
||||
virtual RewriteOptions* NewRewriteOptions();
|
||||
virtual RewriteOptions* NewRewriteOptionsForQuery();
|
||||
virtual ServerContext* NewDecodingServerContext();
|
||||
// Check resolver configured or not.
|
||||
bool CheckResolver();
|
||||
|
||||
// Initializes all the statistics objects created transitively by
|
||||
// NgxRewriteDriverFactory, including nginx-specific and
|
||||
// platform-independent statistics.
|
||||
static void InitStats(Statistics* statistics);
|
||||
NgxServerContext* MakeNgxServerContext(StringPiece hostname, int port);
|
||||
virtual ServerContext* NewServerContext();
|
||||
virtual void ShutDown();
|
||||
|
||||
// Starts pagespeed threads if they've not been started already. Must be
|
||||
// called after the caller has finished any forking it intends to do.
|
||||
void StartThreads();
|
||||
|
||||
void SetServerContextMessageHandler(ServerContext* server_context,
|
||||
ngx_log_t* log);
|
||||
|
||||
NgxMessageHandler* ngx_message_handler() { return ngx_message_handler_; }
|
||||
|
||||
virtual void NonStaticInitStats(Statistics* statistics) {
|
||||
InitStats(statistics);
|
||||
}
|
||||
|
||||
void SetMainConf(NgxRewriteOptions* main_conf);
|
||||
|
||||
void set_resolver(ngx_resolver_t* resolver) {
|
||||
resolver_ = resolver;
|
||||
}
|
||||
void set_resolver_timeout(ngx_msec_t resolver_timeout) {
|
||||
resolver_timeout_ = resolver_timeout == NGX_CONF_UNSET_MSEC ?
|
||||
1000 : resolver_timeout;
|
||||
}
|
||||
bool use_native_fetcher() {
|
||||
return use_native_fetcher_;
|
||||
}
|
||||
void set_use_native_fetcher(bool x) {
|
||||
use_native_fetcher_ = x;
|
||||
}
|
||||
int native_fetcher_max_keepalive_requests() {
|
||||
return native_fetcher_max_keepalive_requests_;
|
||||
}
|
||||
void set_native_fetcher_max_keepalive_requests(int x) {
|
||||
native_fetcher_max_keepalive_requests_ = x;
|
||||
}
|
||||
ProcessScriptVariablesMode process_script_variables() {
|
||||
return process_script_variables_mode_;
|
||||
}
|
||||
|
||||
void LoggingInit(ngx_log_t* log, bool may_install_crash_handler);
|
||||
|
||||
virtual void ShutDownMessageHandlers();
|
||||
|
||||
virtual void SetCircularBuffer(SharedCircularBuffer* buffer);
|
||||
|
||||
bool SetProcessScriptVariables(ProcessScriptVariablesMode mode) {
|
||||
if (!process_script_variables_set_) {
|
||||
process_script_variables_mode_ = mode;
|
||||
process_script_variables_set_ = true;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
virtual void PrepareForkedProcess(const char* name);
|
||||
|
||||
virtual void NameProcess(const char* name);
|
||||
|
||||
private:
|
||||
Timer* timer_;
|
||||
|
||||
bool threads_started_;
|
||||
NgxMessageHandler* ngx_message_handler_;
|
||||
NgxMessageHandler* ngx_html_parse_message_handler_;
|
||||
|
||||
std::vector<NgxUrlAsyncFetcher*> ngx_url_async_fetchers_;
|
||||
ngx_log_t* log_;
|
||||
ngx_msec_t resolver_timeout_;
|
||||
ngx_resolver_t* resolver_;
|
||||
bool use_native_fetcher_;
|
||||
int native_fetcher_max_keepalive_requests_;
|
||||
|
||||
typedef std::set<NgxMessageHandler*> NgxMessageHandlerSet;
|
||||
NgxMessageHandlerSet server_context_message_handlers_;
|
||||
|
||||
// Owned by the superclass.
|
||||
// TODO(jefftk): merge the nginx and apache ways of doing this.
|
||||
SharedCircularBuffer* ngx_shared_circular_buffer_;
|
||||
|
||||
GoogleString hostname_;
|
||||
int port_;
|
||||
ProcessScriptVariablesMode process_script_variables_mode_;
|
||||
bool process_script_variables_set_;
|
||||
bool shut_down_;
|
||||
|
||||
DISALLOW_COPY_AND_ASSIGN(NgxRewriteDriverFactory);
|
||||
};
|
||||
|
||||
} // namespace net_instaweb
|
||||
|
||||
#endif // NGX_REWRITE_DRIVER_FACTORY_H_
|
||||
552
ngx_pagespeed-latest-beta/src/ngx_rewrite_options.cc
Normal file
552
ngx_pagespeed-latest-beta/src/ngx_rewrite_options.cc
Normal file
@@ -0,0 +1,552 @@
|
||||
/*
|
||||
* Copyright 2012 Google Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Author: jefftk@google.com (Jeff Kaufman)
|
||||
|
||||
#include "ngx_rewrite_options.h"
|
||||
|
||||
extern "C" {
|
||||
#include <ngx_config.h>
|
||||
#include <ngx_core.h>
|
||||
#include <ngx_http.h>
|
||||
}
|
||||
|
||||
#include "ngx_pagespeed.h"
|
||||
#include "ngx_rewrite_driver_factory.h"
|
||||
|
||||
#include "net/instaweb/public/version.h"
|
||||
#include "net/instaweb/rewriter/public/file_load_policy.h"
|
||||
#include "net/instaweb/rewriter/public/rewrite_options.h"
|
||||
#include "pagespeed/kernel/base/message_handler.h"
|
||||
#include "pagespeed/kernel/base/timer.h"
|
||||
#include "pagespeed/system/system_caches.h"
|
||||
|
||||
namespace net_instaweb {
|
||||
|
||||
namespace {
|
||||
|
||||
const char kStatisticsPath[] = "StatisticsPath";
|
||||
const char kGlobalStatisticsPath[] = "GlobalStatisticsPath";
|
||||
const char kConsolePath[] = "ConsolePath";
|
||||
const char kMessagesPath[] = "MessagesPath";
|
||||
const char kAdminPath[] = "AdminPath";
|
||||
const char kGlobalAdminPath[] = "GlobalAdminPath";
|
||||
|
||||
// These options are copied from mod_instaweb.cc, where APACHE_CONFIG_OPTIONX
|
||||
// indicates that they can not be set at the directory/location level. They set
|
||||
// options in the RewriteDriverFactory, so they're entirely global and do not
|
||||
// appear in RewriteOptions. They are not alphabetized on purpose, but rather
|
||||
// left in the same order as in mod_instaweb.cc in case we end up needing to
|
||||
// compare.
|
||||
// TODO(oschaaf): this duplication is a short term solution.
|
||||
const char* const server_only_options[] = {
|
||||
"FetcherTimeoutMs",
|
||||
"FetchProxy",
|
||||
"ForceCaching",
|
||||
"GeneratedFilePrefix",
|
||||
"ImgMaxRewritesAtOnce",
|
||||
"InheritVHostConfig",
|
||||
"InstallCrashHandler",
|
||||
"MessageBufferSize",
|
||||
"NumRewriteThreads",
|
||||
"NumExpensiveRewriteThreads",
|
||||
"StaticAssetPrefix",
|
||||
"TrackOriginalContentLength",
|
||||
"UsePerVHostStatistics", // TODO(anupama): What to do about "No longer used"
|
||||
"BlockingRewriteRefererUrls",
|
||||
"CreateSharedMemoryMetadataCache",
|
||||
"LoadFromFile",
|
||||
"LoadFromFileMatch",
|
||||
"LoadFromFileRule",
|
||||
"LoadFromFileRuleMatch",
|
||||
"UseNativeFetcher",
|
||||
"NativeFetcherMaxKeepaliveRequests"
|
||||
};
|
||||
|
||||
// Options that can only be used in the main (http) option scope.
|
||||
const char* const main_only_options[] = {
|
||||
"UseNativeFetcher",
|
||||
"NativeFetcherMaxKeepaliveRequests"
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
RewriteOptions::Properties* NgxRewriteOptions::ngx_properties_ = NULL;
|
||||
|
||||
NgxRewriteOptions::NgxRewriteOptions(const StringPiece& description,
|
||||
ThreadSystem* thread_system)
|
||||
: SystemRewriteOptions(description, thread_system) {
|
||||
Init();
|
||||
}
|
||||
|
||||
NgxRewriteOptions::NgxRewriteOptions(ThreadSystem* thread_system)
|
||||
: SystemRewriteOptions(thread_system) {
|
||||
Init();
|
||||
}
|
||||
|
||||
void NgxRewriteOptions::Init() {
|
||||
DCHECK(ngx_properties_ != NULL)
|
||||
<< "Call NgxRewriteOptions::Initialize() before construction";
|
||||
clear_inherited_scripts_ = false;
|
||||
InitializeOptions(ngx_properties_);
|
||||
}
|
||||
|
||||
void NgxRewriteOptions::AddProperties() {
|
||||
// Nginx-specific options.
|
||||
add_ngx_option(
|
||||
"", &NgxRewriteOptions::statistics_path_, "nsp", kStatisticsPath,
|
||||
kServerScope, "Set the statistics path. Ex: /ngx_pagespeed_statistics",
|
||||
false);
|
||||
add_ngx_option(
|
||||
"", &NgxRewriteOptions::global_statistics_path_, "ngsp",
|
||||
kGlobalStatisticsPath, kProcessScopeStrict,
|
||||
"Set the global statistics path. Ex: /ngx_pagespeed_global_statistics",
|
||||
false);
|
||||
add_ngx_option(
|
||||
"", &NgxRewriteOptions::console_path_, "ncp", kConsolePath, kServerScope,
|
||||
"Set the console path. Ex: /pagespeed_console", false);
|
||||
add_ngx_option(
|
||||
"", &NgxRewriteOptions::messages_path_, "nmp", kMessagesPath,
|
||||
kServerScope, "Set the messages path. Ex: /ngx_pagespeed_message",
|
||||
false);
|
||||
add_ngx_option(
|
||||
"", &NgxRewriteOptions::admin_path_, "nap", kAdminPath,
|
||||
kServerScope, "Set the admin path. Ex: /pagespeed_admin", false);
|
||||
add_ngx_option(
|
||||
"", &NgxRewriteOptions::global_admin_path_, "ngap", kGlobalAdminPath,
|
||||
kProcessScopeStrict,
|
||||
"Set the global admin path. Ex: /pagespeed_global_admin",
|
||||
false);
|
||||
|
||||
MergeSubclassProperties(ngx_properties_);
|
||||
|
||||
// Default properties are global but to set them the current API requires
|
||||
// a RewriteOptions instance and we're in a static method.
|
||||
NgxRewriteOptions dummy_config(NULL);
|
||||
dummy_config.set_default_x_header_value(kModPagespeedVersion);
|
||||
}
|
||||
|
||||
void NgxRewriteOptions::Initialize() {
|
||||
if (Properties::Initialize(&ngx_properties_)) {
|
||||
SystemRewriteOptions::Initialize();
|
||||
AddProperties();
|
||||
}
|
||||
}
|
||||
|
||||
void NgxRewriteOptions::Terminate() {
|
||||
if (Properties::Terminate(&ngx_properties_)) {
|
||||
SystemRewriteOptions::Terminate();
|
||||
}
|
||||
}
|
||||
|
||||
bool NgxRewriteOptions::IsDirective(StringPiece config_directive,
|
||||
StringPiece compare_directive) {
|
||||
return StringCaseEqual(config_directive, compare_directive);
|
||||
}
|
||||
|
||||
RewriteOptions::OptionScope NgxRewriteOptions::GetOptionScope(
|
||||
StringPiece option_name) {
|
||||
ngx_uint_t i;
|
||||
ngx_uint_t size = sizeof(main_only_options) / sizeof(char*);
|
||||
for (i = 0; i < size; i++) {
|
||||
if (StringCaseEqual(main_only_options[i], option_name)) {
|
||||
return kProcessScopeStrict;
|
||||
}
|
||||
}
|
||||
|
||||
size = sizeof(server_only_options) / sizeof(char*);
|
||||
for (i = 0; i < size; i++) {
|
||||
if (StringCaseEqual(server_only_options[i], option_name)) {
|
||||
return kServerScope;
|
||||
}
|
||||
}
|
||||
|
||||
// This could be made more efficient if RewriteOptions provided a map allowing
|
||||
// access of options by their name. It's not too much of a worry at present
|
||||
// since this is just during initialization.
|
||||
for (OptionBaseVector::const_iterator it = all_options().begin();
|
||||
it != all_options().end(); ++it) {
|
||||
RewriteOptions::OptionBase* option = *it;
|
||||
if (option->option_name() == option_name) {
|
||||
// We treat kLegacyProcessScope as kProcessScopeStrict, failing to start
|
||||
// if an option is out of place.
|
||||
return option->scope() == kLegacyProcessScope ? kProcessScopeStrict
|
||||
: option->scope();
|
||||
}
|
||||
}
|
||||
return kDirectoryScope;
|
||||
}
|
||||
|
||||
RewriteOptions::OptionSettingResult NgxRewriteOptions::ParseAndSetOptions0(
|
||||
StringPiece directive, GoogleString* msg, MessageHandler* handler) {
|
||||
if (IsDirective(directive, "on")) {
|
||||
set_enabled(RewriteOptions::kEnabledOn);
|
||||
} else if (IsDirective(directive, "off")) {
|
||||
set_enabled(RewriteOptions::kEnabledOff);
|
||||
} else if (IsDirective(directive, "unplugged")) {
|
||||
set_enabled(RewriteOptions::kEnabledUnplugged);
|
||||
} else {
|
||||
return RewriteOptions::kOptionNameUnknown;
|
||||
}
|
||||
return RewriteOptions::kOptionOk;
|
||||
}
|
||||
|
||||
RewriteOptions::OptionSettingResult
|
||||
NgxRewriteOptions::ParseAndSetOptionFromName1(
|
||||
StringPiece name, StringPiece arg,
|
||||
GoogleString* msg, MessageHandler* handler) {
|
||||
// FileCachePath needs error checking.
|
||||
if (StringCaseEqual(name, kFileCachePath)) {
|
||||
if (!StringCaseStartsWith(arg, "/")) {
|
||||
*msg = "must start with a slash";
|
||||
return RewriteOptions::kOptionValueInvalid;
|
||||
}
|
||||
}
|
||||
|
||||
return SystemRewriteOptions::ParseAndSetOptionFromName1(
|
||||
name, arg, msg, handler);
|
||||
}
|
||||
|
||||
template <class DriverFactoryT>
|
||||
RewriteOptions::OptionSettingResult ParseAndSetOptionHelper(
|
||||
StringPiece option_value,
|
||||
DriverFactoryT* driver_factory,
|
||||
void (DriverFactoryT::*set_option_method)(bool)) {
|
||||
bool parsed_value;
|
||||
if (StringCaseEqual(option_value, "on") ||
|
||||
StringCaseEqual(option_value, "true")) {
|
||||
parsed_value = true;
|
||||
} else if (StringCaseEqual(option_value, "off") ||
|
||||
StringCaseEqual(option_value, "false")) {
|
||||
parsed_value = false;
|
||||
} else {
|
||||
return RewriteOptions::kOptionValueInvalid;
|
||||
}
|
||||
|
||||
(driver_factory->*set_option_method)(parsed_value);
|
||||
return RewriteOptions::kOptionOk;
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
const char* ps_error_string_for_option(
|
||||
ngx_pool_t* pool, StringPiece directive, StringPiece warning) {
|
||||
GoogleString msg =
|
||||
StrCat("\"", directive, "\" ", warning);
|
||||
char* s = string_piece_to_pool_string(pool, msg);
|
||||
if (s == NULL) {
|
||||
return "failed to allocate memory";
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
// Very similar to apache/mod_instaweb::ParseDirective.
|
||||
const char* NgxRewriteOptions::ParseAndSetOptions(
|
||||
StringPiece* args, int n_args, ngx_pool_t* pool, MessageHandler* handler,
|
||||
NgxRewriteDriverFactory* driver_factory,
|
||||
RewriteOptions::OptionScope scope, ngx_conf_t* cf,
|
||||
ProcessScriptVariablesMode script_mode) {
|
||||
CHECK_GE(n_args, 1);
|
||||
|
||||
StringPiece directive = args[0];
|
||||
|
||||
// Remove initial "ModPagespeed" if there is one.
|
||||
StringPiece mod_pagespeed("ModPagespeed");
|
||||
if (StringCaseStartsWith(directive, mod_pagespeed)) {
|
||||
directive.remove_prefix(mod_pagespeed.size());
|
||||
}
|
||||
|
||||
if (GetOptionScope(directive) > scope) {
|
||||
return ps_error_string_for_option(
|
||||
pool, directive, "cannot be set at this scope.");
|
||||
}
|
||||
|
||||
bool compile_scripts = false;
|
||||
|
||||
if (script_mode != ProcessScriptVariablesMode::kOff) {
|
||||
// In the old mode we only allowed a few, so restrict to those.
|
||||
compile_scripts =
|
||||
StringCaseStartsWith(directive, "LoadFromFile") ||
|
||||
StringCaseEqual(directive, "EnableFilters") ||
|
||||
StringCaseEqual(directive, "DisableFilters") ||
|
||||
StringCaseEqual(directive, "DownstreamCachePurgeLocationPrefix") ||
|
||||
StringCaseEqual(directive, "DownstreamCachePurgeMethod") ||
|
||||
StringCaseEqual(directive,
|
||||
"DownstreamCacheRewrittenPercentageThreshold") ||
|
||||
StringCaseEqual(directive, "ShardDomain");
|
||||
// In the new behaviour we also allow scripting of query- and directory-
|
||||
// scoped options.
|
||||
compile_scripts |=
|
||||
script_mode == ProcessScriptVariablesMode::kAll &&
|
||||
(GetOptionScope(directive) <= RewriteOptions::kDirectoryScope ||
|
||||
(StringCaseEqual(directive, "Allow") ||
|
||||
StringCaseEqual(directive, "BlockingRewriteRefererUrls") ||
|
||||
StringCaseEqual(directive, "Disallow") ||
|
||||
StringCaseEqual(directive, "DistributableFilters") ||
|
||||
StringCaseEqual(directive, "Domain") ||
|
||||
StringCaseEqual(directive, "ExperimentVariable") ||
|
||||
StringCaseEqual(directive, "ExperimentSpec") ||
|
||||
StringCaseEqual(directive, "ForbidFilters") ||
|
||||
StringCaseEqual(directive, "RetainComment") ||
|
||||
StringCaseEqual(directive, "CustomFetchHeader") ||
|
||||
StringCaseEqual(directive, "MapOriginDomain") ||
|
||||
StringCaseEqual(directive, "MapProxyDomain") ||
|
||||
StringCaseEqual(directive, "MapRewriteDomain") ||
|
||||
StringCaseEqual(directive, "UrlValuedAttribute") ||
|
||||
StringCaseEqual(directive, "Library")));
|
||||
}
|
||||
|
||||
ScriptLine* script_line;
|
||||
script_line = NULL;
|
||||
|
||||
if (n_args == 1 && StringCaseEqual(directive, "ClearInheritedScripts")) {
|
||||
clear_inherited_scripts_ = true;
|
||||
return NGX_CONF_OK;
|
||||
}
|
||||
|
||||
if (compile_scripts) {
|
||||
CHECK(cf != NULL);
|
||||
int i;
|
||||
// Skip the first arg which is always 'pagespeed'
|
||||
for (i = 1; i < n_args; i++) {
|
||||
ngx_str_t script_source;
|
||||
|
||||
script_source.len = args[i].as_string().length();
|
||||
std::string tmp = args[i].as_string();
|
||||
script_source.data = reinterpret_cast<u_char*>(
|
||||
const_cast<char*>(tmp.c_str()));
|
||||
|
||||
if (ngx_http_script_variables_count(&script_source) > 0) {
|
||||
ngx_http_script_compile_t* sc =
|
||||
reinterpret_cast<ngx_http_script_compile_t*>(
|
||||
ngx_pcalloc(cf->pool, sizeof(ngx_http_script_compile_t)));
|
||||
sc->cf = cf;
|
||||
sc->source = &script_source;
|
||||
sc->lengths = reinterpret_cast<ngx_array_t**>(
|
||||
ngx_pcalloc(cf->pool, sizeof(ngx_array_t*)));
|
||||
sc->values = reinterpret_cast<ngx_array_t**>(
|
||||
ngx_pcalloc(cf->pool, sizeof(ngx_array_t*)));
|
||||
sc->variables = 1;
|
||||
sc->complete_lengths = 1;
|
||||
sc->complete_values = 1;
|
||||
if (ngx_http_script_compile(sc) != NGX_OK) {
|
||||
return ps_error_string_for_option(
|
||||
pool, directive, "Failed to compile script variables");
|
||||
} else {
|
||||
if (script_line == NULL) {
|
||||
script_line = new ScriptLine(args, n_args, scope);
|
||||
}
|
||||
script_line->AddScriptAndArgIndex(sc, i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (script_line != NULL) {
|
||||
script_lines_.push_back(RefCountedPtr<ScriptLine>(script_line));
|
||||
// We have found script variables in the current configuration line, and
|
||||
// prepared the associated rewriteoptions for that.
|
||||
// We will defer parsing, validation and processing of this line to
|
||||
// request time. That means we are done handling this configuration line.
|
||||
return NGX_CONF_OK;
|
||||
}
|
||||
}
|
||||
|
||||
GoogleString msg;
|
||||
OptionSettingResult result;
|
||||
if (n_args == 1) {
|
||||
result = ParseAndSetOptions0(directive, &msg, handler);
|
||||
} else if (n_args == 2) {
|
||||
StringPiece arg = args[1];
|
||||
if (IsDirective(directive, "UseNativeFetcher")) {
|
||||
result = ParseAndSetOptionHelper<NgxRewriteDriverFactory>(
|
||||
arg, driver_factory,
|
||||
&NgxRewriteDriverFactory::set_use_native_fetcher);
|
||||
} else if (IsDirective(directive, "NativeFetcherMaxKeepaliveRequests")) {
|
||||
int max_keepalive_requests;
|
||||
if (StringToInt(arg, &max_keepalive_requests) &&
|
||||
max_keepalive_requests > 0) {
|
||||
driver_factory->set_native_fetcher_max_keepalive_requests(
|
||||
max_keepalive_requests);
|
||||
result = RewriteOptions::kOptionOk;
|
||||
} else {
|
||||
result = RewriteOptions::kOptionValueInvalid;
|
||||
}
|
||||
} else if (StringCaseEqual("ProcessScriptVariables", args[0])) {
|
||||
if (scope == RewriteOptions::kProcessScopeStrict) {
|
||||
ProcessScriptVariablesMode mode;
|
||||
if (StringCaseEqual(arg, "all")) {
|
||||
mode = ProcessScriptVariablesMode::kAll;
|
||||
} else if (StringCaseEqual(arg, "on")) {
|
||||
mode = ProcessScriptVariablesMode::kLegacyRestricted;
|
||||
} else if (StringCaseEqual(arg, "off")) {
|
||||
mode = ProcessScriptVariablesMode::kOff;
|
||||
} else {
|
||||
return const_cast<char*>(
|
||||
"pagespeed ProcessScriptVariables: invalid value");
|
||||
}
|
||||
if (driver_factory->SetProcessScriptVariables(mode)) {
|
||||
result = RewriteOptions::kOptionOk;
|
||||
} else {
|
||||
return const_cast<char*>(
|
||||
"pagespeed ProcessScriptVariables: can only be set once");
|
||||
}
|
||||
} else {
|
||||
return const_cast<char*>(
|
||||
"ProcessScriptVariables is only allowed at the top level");
|
||||
}
|
||||
} else {
|
||||
result = ParseAndSetOptionFromName1(directive, arg, &msg, handler);
|
||||
if (result == RewriteOptions::kOptionNameUnknown) {
|
||||
result = driver_factory->ParseAndSetOption1(
|
||||
directive,
|
||||
arg,
|
||||
scope >= RewriteOptions::kLegacyProcessScope,
|
||||
&msg,
|
||||
handler);
|
||||
}
|
||||
}
|
||||
} else if (n_args == 3) {
|
||||
result = ParseAndSetOptionFromName2(directive, args[1], args[2],
|
||||
&msg, handler);
|
||||
if (result == RewriteOptions::kOptionNameUnknown) {
|
||||
result = driver_factory->ParseAndSetOption2(
|
||||
directive,
|
||||
args[1],
|
||||
args[2],
|
||||
scope >= RewriteOptions::kLegacyProcessScope,
|
||||
&msg,
|
||||
handler);
|
||||
}
|
||||
} else if (n_args == 4) {
|
||||
result = ParseAndSetOptionFromName3(
|
||||
directive, args[1], args[2], args[3], &msg, handler);
|
||||
} else {
|
||||
result = RewriteOptions::kOptionNameUnknown;
|
||||
}
|
||||
|
||||
switch (result) {
|
||||
case RewriteOptions::kOptionOk:
|
||||
return NGX_CONF_OK;
|
||||
case RewriteOptions::kOptionNameUnknown:
|
||||
return ps_error_string_for_option(
|
||||
pool, directive, "not recognized or too many arguments");
|
||||
case RewriteOptions::kOptionValueInvalid: {
|
||||
GoogleString full_directive;
|
||||
for (int i = 0 ; i < n_args ; i++) {
|
||||
StrAppend(&full_directive, i == 0 ? "" : " ", args[i]);
|
||||
}
|
||||
return ps_error_string_for_option(pool, full_directive, msg);
|
||||
}
|
||||
}
|
||||
|
||||
CHECK(false);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// Execute all entries in the script_lines vector, and hand the result off to
|
||||
// ParseAndSetOptions to obtain the final option values.
|
||||
bool NgxRewriteOptions::ExecuteScriptVariables(
|
||||
ngx_http_request_t* r, MessageHandler* handler,
|
||||
NgxRewriteDriverFactory* driver_factory) {
|
||||
bool script_error = false;
|
||||
|
||||
if (script_lines_.size() > 0) {
|
||||
std::vector<RefCountedPtr<ScriptLine> >::iterator it;
|
||||
for (it = script_lines_.begin() ; it != script_lines_.end(); ++it) {
|
||||
ScriptLine* script_line = it->get();
|
||||
StringPiece args[NGX_PAGESPEED_MAX_ARGS];
|
||||
std::vector<ScriptArgIndex*>::iterator cs_it;
|
||||
int i;
|
||||
|
||||
for (i = 0; i < script_line->n_args(); i++) {
|
||||
args[i] = script_line->args()[i];
|
||||
}
|
||||
|
||||
for (cs_it = script_line->data().begin();
|
||||
cs_it != script_line->data().end(); cs_it++) {
|
||||
ngx_http_script_compile_t* script;
|
||||
ngx_array_t* values;
|
||||
ngx_array_t* lengths;
|
||||
ngx_str_t value;
|
||||
|
||||
script = (*cs_it)->script();
|
||||
lengths = *script->lengths;
|
||||
values = *script->values;
|
||||
|
||||
if (ngx_http_script_run(r, &value, lengths->elts, 0, values->elts)
|
||||
== NULL) {
|
||||
handler->Message(kError, "ngx_http_script_run error");
|
||||
script_error = true;
|
||||
break;
|
||||
} else {
|
||||
args[(*cs_it)->index()] = str_to_string_piece(value);
|
||||
}
|
||||
}
|
||||
|
||||
const char* status = ParseAndSetOptions(args, script_line->n_args(),
|
||||
r->pool, handler, driver_factory, script_line->scope(), NULL /*cf*/,
|
||||
ProcessScriptVariablesMode::kOff);
|
||||
|
||||
if (status != NULL) {
|
||||
script_error = true;
|
||||
handler->Message(kWarning,
|
||||
"Error setting option value from script: '%s'", status);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (script_error) {
|
||||
handler->Message(kWarning,
|
||||
"Script error(s) in configuration, disabling optimization");
|
||||
set_enabled(RewriteOptions::kEnabledOff);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void NgxRewriteOptions::CopyScriptLinesTo(
|
||||
NgxRewriteOptions* destination) const {
|
||||
destination->script_lines_ = script_lines_;
|
||||
}
|
||||
|
||||
void NgxRewriteOptions::AppendScriptLinesTo(
|
||||
NgxRewriteOptions* destination) const {
|
||||
destination->script_lines_.insert(destination->script_lines_.end(),
|
||||
script_lines_.begin(), script_lines_.end());
|
||||
}
|
||||
|
||||
NgxRewriteOptions* NgxRewriteOptions::Clone() const {
|
||||
NgxRewriteOptions* options = new NgxRewriteOptions(
|
||||
StrCat("cloned from ", description()), thread_system());
|
||||
this->CopyScriptLinesTo(options);
|
||||
options->Merge(*this);
|
||||
return options;
|
||||
}
|
||||
|
||||
const NgxRewriteOptions* NgxRewriteOptions::DynamicCast(
|
||||
const RewriteOptions* instance) {
|
||||
return dynamic_cast<const NgxRewriteOptions*>(instance);
|
||||
}
|
||||
|
||||
NgxRewriteOptions* NgxRewriteOptions::DynamicCast(RewriteOptions* instance) {
|
||||
return dynamic_cast<NgxRewriteOptions*>(instance);
|
||||
}
|
||||
|
||||
} // namespace net_instaweb
|
||||
248
ngx_pagespeed-latest-beta/src/ngx_rewrite_options.h
Normal file
248
ngx_pagespeed-latest-beta/src/ngx_rewrite_options.h
Normal file
@@ -0,0 +1,248 @@
|
||||
/*
|
||||
* Copyright 2012 Google Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Author: jefftk@google.com (Jeff Kaufman)
|
||||
|
||||
// Manage configuration for pagespeed. Compare to ApacheConfig.
|
||||
|
||||
#ifndef NGX_REWRITE_OPTIONS_H_
|
||||
#define NGX_REWRITE_OPTIONS_H_
|
||||
|
||||
extern "C" {
|
||||
#include <ngx_config.h>
|
||||
#include <ngx_core.h>
|
||||
#include <ngx_http.h>
|
||||
}
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "ngx_rewrite_driver_factory.h"
|
||||
|
||||
#include "net/instaweb/rewriter/public/rewrite_options.h"
|
||||
#include "pagespeed/kernel/base/message_handler.h"
|
||||
#include "pagespeed/kernel/base/ref_counted_ptr.h"
|
||||
#include "pagespeed/kernel/base/stl_util.h" // for STLDeleteElements
|
||||
#include "pagespeed/system/system_rewrite_options.h"
|
||||
|
||||
#define NGX_PAGESPEED_MAX_ARGS 10
|
||||
|
||||
namespace net_instaweb {
|
||||
|
||||
class NgxRewriteDriverFactory;
|
||||
|
||||
class ScriptArgIndex {
|
||||
public:
|
||||
explicit ScriptArgIndex(ngx_http_script_compile_t* script, int index)
|
||||
: script_(script), index_(index) {
|
||||
CHECK(script != NULL);
|
||||
CHECK(index > 0 && index < NGX_PAGESPEED_MAX_ARGS);
|
||||
}
|
||||
|
||||
virtual ~ScriptArgIndex() {}
|
||||
|
||||
ngx_http_script_compile_t* script() { return script_; }
|
||||
int index() { return index_; }
|
||||
|
||||
private:
|
||||
// Not owned.
|
||||
ngx_http_script_compile_t* script_;
|
||||
int index_;
|
||||
};
|
||||
|
||||
// Refcounted, because the ScriptArgIndexes inside data_ can be shared between
|
||||
// different rewriteoptions.
|
||||
class ScriptLine : public RefCounted<ScriptLine> {
|
||||
public:
|
||||
explicit ScriptLine(StringPiece* args, int n_args,
|
||||
RewriteOptions::OptionScope scope)
|
||||
: n_args_(n_args),
|
||||
scope_(scope) {
|
||||
|
||||
for (int i = 0; i < n_args; i++) {
|
||||
args_[i] = args[i];
|
||||
}
|
||||
}
|
||||
|
||||
virtual ~ScriptLine() {
|
||||
STLDeleteElements(&data_);
|
||||
data_.clear();
|
||||
}
|
||||
|
||||
void AddScriptAndArgIndex(ngx_http_script_compile_t* script,
|
||||
int script_index) {
|
||||
CHECK(script != NULL);
|
||||
CHECK(script_index < NGX_PAGESPEED_MAX_ARGS);
|
||||
data_.push_back(new ScriptArgIndex(script, script_index));
|
||||
}
|
||||
|
||||
int n_args() { return n_args_;}
|
||||
StringPiece* args() { return args_;}
|
||||
RewriteOptions::OptionScope scope() { return scope_; }
|
||||
std::vector<ScriptArgIndex*>& data() {
|
||||
return data_;
|
||||
}
|
||||
|
||||
private:
|
||||
StringPiece args_[NGX_PAGESPEED_MAX_ARGS];
|
||||
int n_args_;
|
||||
RewriteOptions::OptionScope scope_;
|
||||
std::vector<ScriptArgIndex*> data_;
|
||||
|
||||
DISALLOW_COPY_AND_ASSIGN(ScriptLine);
|
||||
};
|
||||
|
||||
class NgxRewriteOptions : public SystemRewriteOptions {
|
||||
public:
|
||||
// See rewrite_options::Initialize and ::Terminate
|
||||
static void Initialize();
|
||||
static void Terminate();
|
||||
|
||||
NgxRewriteOptions(const StringPiece& description,
|
||||
ThreadSystem* thread_system);
|
||||
explicit NgxRewriteOptions(ThreadSystem* thread_system);
|
||||
virtual ~NgxRewriteOptions() { }
|
||||
|
||||
// args is an array of n_args StringPieces together representing a directive.
|
||||
// For example:
|
||||
// ["RewriteLevel", "PassThrough"]
|
||||
// or
|
||||
// ["EnableFilters", "combine_css,extend_cache,rewrite_images"]
|
||||
// or
|
||||
// ["ShardDomain", "example.com", "s1.example.com,s2.example.com"]
|
||||
// Apply the directive, returning NGX_CONF_OK on success or an error message
|
||||
// on failure.
|
||||
//
|
||||
// pool is a memory pool for allocating error strings.
|
||||
// cf is only required when compile_scripts is true
|
||||
// when compile_scripts is true, the rewrite_options will be prepared
|
||||
// for replacing any script $variables encountered in args. when false,
|
||||
// script variables will be substituted using the prepared rewrite options.
|
||||
const char* ParseAndSetOptions(
|
||||
StringPiece* args, int n_args, ngx_pool_t* pool, MessageHandler* handler,
|
||||
NgxRewriteDriverFactory* driver_factory, OptionScope scope,
|
||||
ngx_conf_t* cf, ProcessScriptVariablesMode script_mode);
|
||||
bool ExecuteScriptVariables(
|
||||
ngx_http_request_t* r, MessageHandler* handler,
|
||||
NgxRewriteDriverFactory* driver_factory);
|
||||
void CopyScriptLinesTo(NgxRewriteOptions* destination) const;
|
||||
void AppendScriptLinesTo(NgxRewriteOptions* destination) const;
|
||||
|
||||
// Make an identical copy of these options and return it.
|
||||
virtual NgxRewriteOptions* Clone() const;
|
||||
|
||||
// Returns a suitably down cast version of 'instance' if it is an instance
|
||||
// of this class, NULL if not.
|
||||
static const NgxRewriteOptions* DynamicCast(const RewriteOptions* instance);
|
||||
static NgxRewriteOptions* DynamicCast(RewriteOptions* instance);
|
||||
|
||||
const GoogleString& statistics_path() const {
|
||||
return statistics_path_.value();
|
||||
}
|
||||
const GoogleString& global_statistics_path() const {
|
||||
return global_statistics_path_.value();
|
||||
}
|
||||
const GoogleString& console_path() const {
|
||||
return console_path_.value();
|
||||
}
|
||||
const GoogleString& messages_path() const {
|
||||
return messages_path_.value();
|
||||
}
|
||||
const GoogleString& admin_path() const {
|
||||
return admin_path_.value();
|
||||
}
|
||||
const GoogleString& global_admin_path() const {
|
||||
return global_admin_path_.value();
|
||||
}
|
||||
const std::vector<RefCountedPtr<ScriptLine> >& script_lines() const {
|
||||
return script_lines_;
|
||||
}
|
||||
const bool& clear_inherited_scripts() const {
|
||||
return clear_inherited_scripts_;
|
||||
}
|
||||
|
||||
private:
|
||||
// Helper methods for ParseAndSetOptions(). Each can:
|
||||
// - return kOptionNameUnknown and not set msg:
|
||||
// - directive not handled; continue on with other possible
|
||||
// interpretations.
|
||||
// - return kOptionOk and not set msg:
|
||||
// - directive handled, all's well.
|
||||
// - return kOptionValueInvalid and set msg:
|
||||
// - directive handled with an error; return the error to the user.
|
||||
//
|
||||
// msg will be shown to the user on kOptionValueInvalid. While it would be
|
||||
// nice to always use msg and never use the MessageHandler, some option
|
||||
// parsing code in RewriteOptions expects to write to a MessageHandler. If
|
||||
// that happens we put a summary on msg so the user sees something, and the
|
||||
// detailed message goes to their log via handler.
|
||||
OptionSettingResult ParseAndSetOptions0(
|
||||
StringPiece directive, GoogleString* msg, MessageHandler* handler);
|
||||
|
||||
virtual OptionSettingResult ParseAndSetOptionFromName1(
|
||||
StringPiece name, StringPiece arg,
|
||||
GoogleString* msg, MessageHandler* handler);
|
||||
|
||||
// We may want to override 2- and 3-argument versions as well in the future,
|
||||
// but they are not needed yet.
|
||||
|
||||
// Keeps the properties added by this subclass. These are merged into
|
||||
// RewriteOptions::all_properties_ during Initialize().
|
||||
//
|
||||
// RewriteOptions uses static initialization to reduce memory usage and
|
||||
// construction time. All NgxRewriteOptions instances will have the same
|
||||
// Properties, so we can build the list when we initialize the first one.
|
||||
static Properties* ngx_properties_;
|
||||
static void AddProperties();
|
||||
void Init();
|
||||
|
||||
// Add an option to ngx_properties_
|
||||
template<class OptionClass>
|
||||
static void add_ngx_option(typename OptionClass::ValueType default_value,
|
||||
OptionClass NgxRewriteOptions::*offset,
|
||||
const char* id,
|
||||
StringPiece option_name,
|
||||
OptionScope scope,
|
||||
const char* help,
|
||||
bool safe_to_print) {
|
||||
AddProperty(default_value, offset, id, option_name, scope, help,
|
||||
safe_to_print, ngx_properties_);
|
||||
}
|
||||
|
||||
Option<GoogleString> statistics_path_;
|
||||
Option<GoogleString> global_statistics_path_;
|
||||
Option<GoogleString> console_path_;
|
||||
Option<GoogleString> messages_path_;
|
||||
Option<GoogleString> admin_path_;
|
||||
Option<GoogleString> global_admin_path_;
|
||||
|
||||
bool clear_inherited_scripts_;
|
||||
std::vector<RefCountedPtr<ScriptLine> > script_lines_;
|
||||
|
||||
// Helper for ParseAndSetOptions. Returns whether the two directives equal,
|
||||
// ignoring case.
|
||||
bool IsDirective(StringPiece config_directive, StringPiece compare_directive);
|
||||
|
||||
// Returns a given option's scope.
|
||||
RewriteOptions::OptionScope GetOptionScope(StringPiece option_name);
|
||||
|
||||
// TODO(jefftk): support fetch proxy in server and location blocks.
|
||||
|
||||
DISALLOW_COPY_AND_ASSIGN(NgxRewriteOptions);
|
||||
};
|
||||
|
||||
} // namespace net_instaweb
|
||||
|
||||
#endif // NGX_REWRITE_OPTIONS_H_
|
||||
98
ngx_pagespeed-latest-beta/src/ngx_server_context.cc
Normal file
98
ngx_pagespeed-latest-beta/src/ngx_server_context.cc
Normal file
@@ -0,0 +1,98 @@
|
||||
/*
|
||||
* Copyright 2012 Google Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Author: jefftk@google.com (Jeff Kaufman)
|
||||
|
||||
#include "ngx_server_context.h"
|
||||
|
||||
extern "C" {
|
||||
#include <ngx_http.h>
|
||||
}
|
||||
|
||||
#include "ngx_pagespeed.h"
|
||||
#include "ngx_message_handler.h"
|
||||
#include "ngx_rewrite_driver_factory.h"
|
||||
#include "ngx_rewrite_options.h"
|
||||
#include "net/instaweb/rewriter/public/rewrite_driver.h"
|
||||
#include "pagespeed/system/add_headers_fetcher.h"
|
||||
#include "pagespeed/system/loopback_route_fetcher.h"
|
||||
#include "pagespeed/system/system_request_context.h"
|
||||
|
||||
namespace net_instaweb {
|
||||
|
||||
NgxServerContext::NgxServerContext(
|
||||
NgxRewriteDriverFactory* factory, StringPiece hostname, int port)
|
||||
: SystemServerContext(factory, hostname, port),
|
||||
ngx_http2_variable_index_(NGX_ERROR) {
|
||||
}
|
||||
|
||||
NgxServerContext::~NgxServerContext() { }
|
||||
|
||||
NgxRewriteOptions* NgxServerContext::config() {
|
||||
return NgxRewriteOptions::DynamicCast(global_options());
|
||||
}
|
||||
|
||||
SystemRequestContext* NgxServerContext::NewRequestContext(
|
||||
ngx_http_request_t* r) {
|
||||
// Based on ngx_http_variable_server_port.
|
||||
bool port_set = false;
|
||||
int local_port = 0;
|
||||
#if (NGX_HAVE_INET6)
|
||||
if (r->connection->local_sockaddr->sa_family == AF_INET6) {
|
||||
local_port = ntohs(reinterpret_cast<struct sockaddr_in6*>(
|
||||
r->connection->local_sockaddr)->sin6_port);
|
||||
port_set = true;
|
||||
}
|
||||
#endif
|
||||
if (!port_set) {
|
||||
local_port = ntohs(reinterpret_cast<struct sockaddr_in*>(
|
||||
r->connection->local_sockaddr)->sin_port);
|
||||
}
|
||||
|
||||
ngx_str_t local_ip;
|
||||
u_char addr[NGX_SOCKADDR_STRLEN];
|
||||
local_ip.len = NGX_SOCKADDR_STRLEN;
|
||||
local_ip.data = addr;
|
||||
ngx_int_t rc = ngx_connection_local_sockaddr(r->connection, &local_ip, 0);
|
||||
if (rc != NGX_OK) {
|
||||
local_ip.len = 0;
|
||||
}
|
||||
|
||||
SystemRequestContext* ctx = new SystemRequestContext(
|
||||
thread_system()->NewMutex(), timer(),
|
||||
ps_determine_host(r), local_port, str_to_string_piece(local_ip));
|
||||
|
||||
// See if http2 is in use.
|
||||
if (ngx_http2_variable_index_ >= 0) {
|
||||
ngx_http_variable_value_t* val =
|
||||
ngx_http_get_indexed_variable(r, ngx_http2_variable_index_);
|
||||
if (val != NULL && val->valid) {
|
||||
StringPiece str_val(reinterpret_cast<char*>(val->data), val->len);
|
||||
if (str_val == "h2" || str_val == "h2c") {
|
||||
ctx->set_using_http2(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ctx;
|
||||
}
|
||||
|
||||
GoogleString NgxServerContext::FormatOption(StringPiece option_name,
|
||||
StringPiece args) {
|
||||
return StrCat("pagespeed ", option_name, " ", args, ";");
|
||||
}
|
||||
|
||||
} // namespace net_instaweb
|
||||
79
ngx_pagespeed-latest-beta/src/ngx_server_context.h
Normal file
79
ngx_pagespeed-latest-beta/src/ngx_server_context.h
Normal file
@@ -0,0 +1,79 @@
|
||||
/*
|
||||
* Copyright 2012 Google Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Author: jefftk@google.com (Jeff Kaufman)
|
||||
|
||||
// Manage pagespeed state across requests. Compare to ApacheResourceManager.
|
||||
|
||||
#ifndef NGX_SERVER_CONTEXT_H_
|
||||
#define NGX_SERVER_CONTEXT_H_
|
||||
|
||||
#include "ngx_message_handler.h"
|
||||
#include "pagespeed/system/system_server_context.h"
|
||||
|
||||
extern "C" {
|
||||
#include <ngx_http.h>
|
||||
}
|
||||
|
||||
namespace net_instaweb {
|
||||
|
||||
class NgxRewriteDriverFactory;
|
||||
class NgxRewriteOptions;
|
||||
class SystemRequestContext;
|
||||
|
||||
class NgxServerContext : public SystemServerContext {
|
||||
public:
|
||||
NgxServerContext(
|
||||
NgxRewriteDriverFactory* factory, StringPiece hostname, int port);
|
||||
virtual ~NgxServerContext();
|
||||
|
||||
// We don't allow ProxyFetch to fetch HTML via MapProxyDomain. We will call
|
||||
// set_trusted_input() on any ProxyFetches we use to transform internal HTML.
|
||||
virtual bool ProxiesHtml() const { return false; }
|
||||
|
||||
// Call only when you need an NgxRewriteOptions. If you don't need
|
||||
// nginx-specific behavior, call global_options() instead which doesn't
|
||||
// downcast.
|
||||
NgxRewriteOptions* config();
|
||||
|
||||
NgxRewriteDriverFactory* ngx_rewrite_driver_factory() { return ngx_factory_; }
|
||||
SystemRequestContext* NewRequestContext(ngx_http_request_t* r);
|
||||
|
||||
NgxMessageHandler* ngx_message_handler() {
|
||||
return dynamic_cast<NgxMessageHandler*>(message_handler());
|
||||
}
|
||||
|
||||
virtual GoogleString FormatOption(StringPiece option_name, StringPiece args);
|
||||
|
||||
void set_ngx_http2_variable_index(ngx_int_t idx) {
|
||||
ngx_http2_variable_index_ = idx;
|
||||
}
|
||||
|
||||
ngx_int_t ngx_http2_variable_index() const {
|
||||
return ngx_http2_variable_index_;
|
||||
}
|
||||
|
||||
private:
|
||||
NgxRewriteDriverFactory* ngx_factory_;
|
||||
// what index the "http2" var is, or NGX_ERROR.
|
||||
ngx_int_t ngx_http2_variable_index_;
|
||||
|
||||
DISALLOW_COPY_AND_ASSIGN(NgxServerContext);
|
||||
};
|
||||
|
||||
} // namespace net_instaweb
|
||||
|
||||
#endif // NGX_SERVER_CONTEXT_H_
|
||||
299
ngx_pagespeed-latest-beta/src/ngx_url_async_fetcher.cc
Normal file
299
ngx_pagespeed-latest-beta/src/ngx_url_async_fetcher.cc
Normal file
@@ -0,0 +1,299 @@
|
||||
/*
|
||||
* Copyright 2012 Google Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Author: x.dinic@gmail.com (Junmin Xiong)
|
||||
|
||||
extern "C" {
|
||||
#include <ngx_http.h>
|
||||
#include <ngx_core.h>
|
||||
}
|
||||
|
||||
#include "ngx_url_async_fetcher.h"
|
||||
#include "ngx_fetch.h"
|
||||
|
||||
#include <vector>
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
#include <list>
|
||||
#include <map>
|
||||
#include <set>
|
||||
|
||||
#include "net/instaweb/http/public/async_fetch.h"
|
||||
#include "net/instaweb/http/public/inflating_fetch.h"
|
||||
#include "net/instaweb/public/version.h"
|
||||
#include "pagespeed/kernel/base/basictypes.h"
|
||||
#include "pagespeed/kernel/base/condvar.h"
|
||||
#include "pagespeed/kernel/base/message_handler.h"
|
||||
#include "pagespeed/kernel/base/pool.h"
|
||||
#include "pagespeed/kernel/base/pool_element.h"
|
||||
#include "pagespeed/kernel/base/statistics.h"
|
||||
#include "pagespeed/kernel/base/string_util.h"
|
||||
#include "pagespeed/kernel/base/thread_system.h"
|
||||
#include "pagespeed/kernel/base/timer.h"
|
||||
#include "pagespeed/kernel/base/writer.h"
|
||||
#include "pagespeed/kernel/http/request_headers.h"
|
||||
#include "pagespeed/kernel/http/response_headers.h"
|
||||
#include "pagespeed/kernel/http/response_headers_parser.h"
|
||||
|
||||
namespace net_instaweb {
|
||||
|
||||
NgxUrlAsyncFetcher::NgxUrlAsyncFetcher(const char* proxy,
|
||||
ngx_log_t* log,
|
||||
ngx_msec_t resolver_timeout,
|
||||
ngx_msec_t fetch_timeout,
|
||||
ngx_resolver_t* resolver,
|
||||
int max_keepalive_requests,
|
||||
ThreadSystem* thread_system,
|
||||
MessageHandler* handler)
|
||||
: fetchers_count_(0),
|
||||
shutdown_(false),
|
||||
track_original_content_length_(false),
|
||||
byte_count_(0),
|
||||
thread_system_(thread_system),
|
||||
message_handler_(handler),
|
||||
mutex_(NULL),
|
||||
max_keepalive_requests_(max_keepalive_requests),
|
||||
event_connection_(NULL) {
|
||||
resolver_timeout_ = resolver_timeout;
|
||||
fetch_timeout_ = fetch_timeout;
|
||||
ngx_memzero(&proxy_, sizeof(proxy_));
|
||||
if (proxy != NULL && *proxy != '\0') {
|
||||
proxy_.url.data = reinterpret_cast<u_char*>(const_cast<char*>(proxy));
|
||||
proxy_.url.len = ngx_strlen(proxy);
|
||||
}
|
||||
mutex_ = thread_system_->NewMutex();
|
||||
log_ = log;
|
||||
pool_ = NULL;
|
||||
resolver_ = resolver;
|
||||
// If init fails, set shutdown_ so no fetches will be attempted.
|
||||
if (!Init(const_cast<ngx_cycle_t*>(ngx_cycle))) {
|
||||
shutdown_ = true;
|
||||
message_handler_->Message(
|
||||
kError, "NgxUrlAsyncFetcher failed to init, fetching disabled.");
|
||||
}
|
||||
}
|
||||
|
||||
NgxUrlAsyncFetcher::~NgxUrlAsyncFetcher() {
|
||||
DCHECK(shutdown_) << "Shut down before destructing NgxUrlAsyncFetcher.";
|
||||
message_handler_->Message(
|
||||
kInfo,
|
||||
"Destruct NgxUrlAsyncFetcher with [%d] active fetchers",
|
||||
ApproximateNumActiveFetches());
|
||||
|
||||
CancelActiveFetches();
|
||||
active_fetches_.DeleteAll();
|
||||
NgxConnection::Terminate();
|
||||
|
||||
if (pool_ != NULL) {
|
||||
ngx_destroy_pool(pool_);
|
||||
pool_ = NULL;
|
||||
}
|
||||
if (mutex_ != NULL) {
|
||||
delete mutex_;
|
||||
mutex_ = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
bool NgxUrlAsyncFetcher::ParseUrl(ngx_url_t* url, ngx_pool_t* pool) {
|
||||
size_t scheme_offset;
|
||||
u_short port;
|
||||
if (ngx_strncasecmp(url->url.data, reinterpret_cast<u_char*>(
|
||||
const_cast<char*>("http://")), 7) == 0) {
|
||||
scheme_offset = 7;
|
||||
port = 80;
|
||||
} else if (ngx_strncasecmp(url->url.data, reinterpret_cast<u_char*>(
|
||||
const_cast<char*>("https://")), 8) == 0) {
|
||||
scheme_offset = 8;
|
||||
port = 443;
|
||||
} else {
|
||||
scheme_offset = 0;
|
||||
port = 80;
|
||||
}
|
||||
|
||||
url->url.data += scheme_offset;
|
||||
url->url.len -= scheme_offset;
|
||||
url->default_port = port;
|
||||
// See: http://lxr.evanmiller.org/http/source/core/ngx_inet.c#L875
|
||||
url->no_resolve = 0;
|
||||
url->uri_part = 1;
|
||||
|
||||
if (ngx_parse_url(pool, url) == NGX_OK) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// If there are still active requests, cancel them.
|
||||
void NgxUrlAsyncFetcher::CancelActiveFetches() {
|
||||
// TODO(oschaaf): this seems tricky, this may end up calling
|
||||
// FetchComplete, modifying the active fetches while we are looping
|
||||
// it
|
||||
for (NgxFetchPool::const_iterator p = active_fetches_.begin(),
|
||||
e = active_fetches_.end(); p != e; ++p) {
|
||||
NgxFetch* fetch = *p;
|
||||
fetch->CallbackDone(false);
|
||||
}
|
||||
}
|
||||
|
||||
// Create the pool for fetcher, create the pipe, add the read event for main
|
||||
// thread. It should be called in the worker process.
|
||||
bool NgxUrlAsyncFetcher::Init(ngx_cycle_t* cycle) {
|
||||
log_ = cycle->log;
|
||||
CHECK(event_connection_ == NULL) << "event connection already set";
|
||||
event_connection_ = new NgxEventConnection(ReadCallback);
|
||||
if (!event_connection_->Init(cycle)) {
|
||||
return false;
|
||||
}
|
||||
if (pool_ == NULL) {
|
||||
pool_ = ngx_create_pool(4096, log_);
|
||||
if (pool_ == NULL) {
|
||||
ngx_log_error(NGX_LOG_ERR, log_, 0,
|
||||
"NgxUrlAsyncFetcher::Init create pool failed");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (proxy_.url.len == 0) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// TODO(oschaaf): shouldn't we do this earlier? Do we need to clean
|
||||
// up when parsing the url failed?
|
||||
if (!ParseUrl(&proxy_, pool_)) {
|
||||
ngx_log_error(NGX_LOG_ERR, log_, 0,
|
||||
"NgxUrlAsyncFetcher::Init parse proxy[%V] failed", &proxy_.url);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void NgxUrlAsyncFetcher::ShutDown() {
|
||||
shutdown_ = true;
|
||||
if (!pending_fetches_.empty()) {
|
||||
for (Pool<NgxFetch>::iterator p = pending_fetches_.begin(),
|
||||
e = pending_fetches_.end(); p != e; p++) {
|
||||
NgxFetch* fetch = *p;
|
||||
fetch->CallbackDone(false);
|
||||
}
|
||||
pending_fetches_.DeleteAll();
|
||||
}
|
||||
|
||||
if (!active_fetches_.empty()) {
|
||||
for (Pool<NgxFetch>::iterator p = active_fetches_.begin(),
|
||||
e = active_fetches_.end(); p != e; p++) {
|
||||
NgxFetch* fetch = *p;
|
||||
fetch->CallbackDone(false);
|
||||
}
|
||||
active_fetches_.Clear();
|
||||
}
|
||||
if (event_connection_ != NULL) {
|
||||
event_connection_->Shutdown();
|
||||
delete event_connection_;
|
||||
event_connection_ = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
// It's called in the rewrite thread. All the fetches are started at
|
||||
// this function. It will notify the main thread to start the fetch job.
|
||||
void NgxUrlAsyncFetcher::Fetch(const GoogleString& url,
|
||||
MessageHandler* message_handler,
|
||||
AsyncFetch* async_fetch) {
|
||||
// Don't accept new fetches when shut down. This flow is also entered when
|
||||
// we did not initialize properly in ::Init().
|
||||
if (shutdown_) {
|
||||
async_fetch->Done(false);
|
||||
return;
|
||||
}
|
||||
async_fetch = EnableInflation(async_fetch);
|
||||
NgxFetch* fetch = new NgxFetch(url, async_fetch,
|
||||
message_handler, log_);
|
||||
ScopedMutex lock(mutex_);
|
||||
pending_fetches_.Add(fetch);
|
||||
|
||||
// TODO(oschaaf): thread safety on written vs shutdown.
|
||||
// It is possible that shutdown() is called after writing an event? In that
|
||||
// case, this could (rarely) fail when it shouldn't.
|
||||
bool written = event_connection_->WriteEvent(this);
|
||||
CHECK(written || shutdown_) << "NgxUrlAsyncFetcher: event write failure";
|
||||
}
|
||||
|
||||
// This is the read event which is called in the main thread.
|
||||
// It will do the real work. Add the work event and start the fetch.
|
||||
void NgxUrlAsyncFetcher::ReadCallback(const ps_event_data& data) {
|
||||
std::vector<NgxFetch*> to_start;
|
||||
NgxUrlAsyncFetcher* fetcher = reinterpret_cast<NgxUrlAsyncFetcher*>(
|
||||
data.sender);
|
||||
|
||||
fetcher->mutex_->Lock();
|
||||
fetcher->completed_fetches_.DeleteAll();
|
||||
|
||||
for (Pool<NgxFetch>::iterator p = fetcher->pending_fetches_.begin(),
|
||||
e = fetcher->pending_fetches_.end(); p != e; p++) {
|
||||
NgxFetch* fetch = *p;
|
||||
to_start.push_back(fetch);
|
||||
}
|
||||
|
||||
fetcher->pending_fetches_.Clear();
|
||||
fetcher->mutex_->Unlock();
|
||||
|
||||
for (size_t i = 0; i < to_start.size(); i++) {
|
||||
fetcher->StartFetch(to_start[i]);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO(oschaaf): return value is ignored.
|
||||
bool NgxUrlAsyncFetcher::StartFetch(NgxFetch* fetch) {
|
||||
mutex_->Lock();
|
||||
active_fetches_.Add(fetch);
|
||||
fetchers_count_++;
|
||||
mutex_->Unlock();
|
||||
|
||||
// Don't initiate the fetch when we are shutting down
|
||||
if (shutdown_) {
|
||||
fetch->CallbackDone(false);
|
||||
return false;
|
||||
}
|
||||
|
||||
bool started = fetch->Start(this);
|
||||
|
||||
if (!started) {
|
||||
message_handler_->Message(kWarning, "Fetch failed to start: %s",
|
||||
fetch->str_url());
|
||||
fetch->CallbackDone(false);
|
||||
}
|
||||
|
||||
return started;
|
||||
}
|
||||
|
||||
void NgxUrlAsyncFetcher::FetchComplete(NgxFetch* fetch) {
|
||||
ScopedMutex lock(mutex_);
|
||||
byte_count_ += fetch->bytes_received();
|
||||
fetchers_count_--;
|
||||
active_fetches_.Remove(fetch);
|
||||
completed_fetches_.Add(fetch);
|
||||
}
|
||||
|
||||
void NgxUrlAsyncFetcher::PrintActiveFetches(MessageHandler* handler) const {
|
||||
for (NgxFetchPool::const_iterator p = active_fetches_.begin(),
|
||||
e = active_fetches_.end(); p != e; ++p) {
|
||||
NgxFetch* fetch = *p;
|
||||
handler->Message(kInfo, "Active fetch: %s", fetch->str_url());
|
||||
}
|
||||
}
|
||||
} // namespace net_instaweb
|
||||
154
ngx_pagespeed-latest-beta/src/ngx_url_async_fetcher.h
Normal file
154
ngx_pagespeed-latest-beta/src/ngx_url_async_fetcher.h
Normal file
@@ -0,0 +1,154 @@
|
||||
/*
|
||||
* Copyright 2012 Google Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Author: x.dinic@gmail.com(Junmin Xiong)
|
||||
//
|
||||
// Fetch the resources asynchronously in Nginx. The fetcher is called in
|
||||
// the rewrite thread.
|
||||
//
|
||||
// It can communicate with Nginx by pipe. One pipe for one fetcher.
|
||||
// When new url fetch comes, Fetcher will add it to the pending queue and
|
||||
// notify the Nginx thread to start the Fetch event. All the events are hooked
|
||||
// in the main thread's epoll structure.
|
||||
|
||||
#ifndef NET_INSTAWEB_NGX_URL_ASYNC_FETCHER_H_
|
||||
#define NET_INSTAWEB_NGX_URL_ASYNC_FETCHER_H_
|
||||
|
||||
extern "C" {
|
||||
#include <ngx_config.h>
|
||||
#include <ngx_core.h>
|
||||
}
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "ngx_event_connection.h"
|
||||
|
||||
#include "net/instaweb/http/public/url_async_fetcher.h"
|
||||
#include "pagespeed/kernel/base/basictypes.h"
|
||||
#include "pagespeed/kernel/base/pool.h"
|
||||
#include "pagespeed/kernel/base/string.h"
|
||||
#include "pagespeed/kernel/base/thread_system.h"
|
||||
|
||||
|
||||
namespace net_instaweb {
|
||||
|
||||
class AsyncFetch;
|
||||
class MessageHandler;
|
||||
class Statistics;
|
||||
class NgxFetch;
|
||||
class Variable;
|
||||
|
||||
class NgxUrlAsyncFetcher : public UrlAsyncFetcher {
|
||||
public:
|
||||
NgxUrlAsyncFetcher(
|
||||
const char* proxy, ngx_log_t* log, ngx_msec_t resolver_timeout,
|
||||
ngx_msec_t fetch_timeout, ngx_resolver_t* resolver,
|
||||
int max_keepalive_requests, ThreadSystem* thread_system,
|
||||
MessageHandler* handler);
|
||||
|
||||
~NgxUrlAsyncFetcher();
|
||||
|
||||
// It should be called in the module init_process callback function. Do some
|
||||
// intializations which can't be done in the master process
|
||||
bool Init(ngx_cycle_t* cycle);
|
||||
|
||||
// shutdown all the fetches.
|
||||
virtual void ShutDown();
|
||||
|
||||
// the read handler in the main thread
|
||||
static void ReadCallback(const ps_event_data& data);
|
||||
|
||||
virtual bool SupportsHttps() const { return false; }
|
||||
|
||||
virtual void Fetch(const GoogleString& url,
|
||||
MessageHandler* message_handler,
|
||||
AsyncFetch* callback);
|
||||
|
||||
bool StartFetch(NgxFetch* fetch);
|
||||
|
||||
// Remove the completed fetch from the active fetch set, and put it into a
|
||||
// completed fetch list to be cleaned up.
|
||||
void FetchComplete(NgxFetch* fetch);
|
||||
void PrintActiveFetches(MessageHandler* handler) const;
|
||||
|
||||
// Indicates that it should track the original content length for
|
||||
// fetched resources.
|
||||
bool track_original_content_length() {
|
||||
return track_original_content_length_;
|
||||
}
|
||||
void set_track_original_content_length(bool x) {
|
||||
track_original_content_length_ = x;
|
||||
}
|
||||
|
||||
typedef Pool<NgxFetch> NgxFetchPool;
|
||||
|
||||
// AnyPendingFetches is accurate only at the time of call; this is
|
||||
// used conservatively during shutdown. It counts fetches that have been
|
||||
// requested by some thread, and can include fetches for which no action
|
||||
// has yet been taken (ie fetches that are not active).
|
||||
virtual bool AnyPendingFetches() {
|
||||
return !active_fetches_.empty();
|
||||
}
|
||||
|
||||
// ApproximateNumActiveFetches can under- or over-count and is used only for
|
||||
// error reporting.
|
||||
int ApproximateNumActiveFetches() {
|
||||
return active_fetches_.size();
|
||||
}
|
||||
|
||||
void CancelActiveFetches();
|
||||
|
||||
// These must be accessed with mutex_ held.
|
||||
bool shutdown() const { return shutdown_; }
|
||||
void set_shutdown(bool s) { shutdown_ = s; }
|
||||
|
||||
|
||||
private:
|
||||
static void TimeoutHandler(ngx_event_t* tev);
|
||||
static bool ParseUrl(ngx_url_t* url, ngx_pool_t* pool);
|
||||
friend class NgxFetch;
|
||||
|
||||
NgxFetchPool active_fetches_;
|
||||
// Add the pending task to this list
|
||||
NgxFetchPool pending_fetches_;
|
||||
NgxFetchPool completed_fetches_;
|
||||
ngx_url_t proxy_;
|
||||
|
||||
int fetchers_count_;
|
||||
bool shutdown_;
|
||||
bool track_original_content_length_;
|
||||
int64 byte_count_;
|
||||
ThreadSystem* thread_system_;
|
||||
MessageHandler* message_handler_;
|
||||
// Protect the member variable in this class
|
||||
// active_fetches, pending_fetches
|
||||
ThreadSystem::CondvarCapableMutex* mutex_;
|
||||
|
||||
ngx_pool_t* pool_;
|
||||
ngx_log_t* log_;
|
||||
ngx_resolver_t* resolver_;
|
||||
int max_keepalive_requests_;
|
||||
ngx_msec_t resolver_timeout_;
|
||||
ngx_msec_t fetch_timeout_;
|
||||
|
||||
NgxEventConnection* event_connection_;
|
||||
|
||||
DISALLOW_COPY_AND_ASSIGN(NgxUrlAsyncFetcher);
|
||||
};
|
||||
|
||||
} // namespace net_instaweb
|
||||
|
||||
#endif // NET_INSTAWEB_NGX_URL_ASYNC_FETCHER_H_$
|
||||
1453
ngx_pagespeed-latest-beta/test/nginx_system_test.sh
Normal file
1453
ngx_pagespeed-latest-beta/test/nginx_system_test.sh
Normal file
File diff suppressed because it is too large
Load Diff
2243
ngx_pagespeed-latest-beta/test/pagespeed_test.conf.template
Normal file
2243
ngx_pagespeed-latest-beta/test/pagespeed_test.conf.template
Normal file
File diff suppressed because it is too large
Load Diff
98
ngx_pagespeed-latest-beta/test/run_tests.sh
Executable file
98
ngx_pagespeed-latest-beta/test/run_tests.sh
Executable file
@@ -0,0 +1,98 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Copyright 2013 Google Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
# Author: jefftk@google.com (Jeff Kaufman)
|
||||
#
|
||||
# Runs ngx_pagespeed system tests.
|
||||
#
|
||||
# Exits with status 0 if all tests pass.
|
||||
# Exits with status 1 immediately if any test fails.
|
||||
# Exits with status 2 if command line args are wrong.
|
||||
#
|
||||
# Usage:
|
||||
# ./run_tests.sh /path/to/mod_pagespeed /path/to/nginx/binary
|
||||
#
|
||||
# By default the test script uses several ports. If you have a port conflict
|
||||
# and need to override one you can do that by setting the relevant environment
|
||||
# variable. For example:
|
||||
# PRIMARY_PORT=1234 ./run_tests.sh /.../mod_pagespeed /.../nginx/binary
|
||||
|
||||
# Normally we test only with the native fetcher off. Set
|
||||
# TEST_NATIVE_FETCHER=true to also test the native fetcher, set
|
||||
# TEST_SERF_FETCHER=false to skip the serf fetcher.
|
||||
TEST_NATIVE_FETCHER=${TEST_NATIVE_FETCHER:-false}
|
||||
TEST_SERF_FETCHER=${TEST_SERF_FETCHER:-true}
|
||||
|
||||
# Normally we actually run the tests, but you might only want us to set up nginx
|
||||
# for you so you can do manual testing. If so, set RUN_TESTS=false and this
|
||||
# will exit after configuring and starting nginx.
|
||||
RUN_TESTS=${RUN_TESTS:-true}
|
||||
|
||||
# To run tests with valgrind, set the environment variable USE_VALGRIND to
|
||||
# true.
|
||||
USE_VALGRIND=${USE_VALGRIND:-false}
|
||||
|
||||
if [ "$#" -ne 2 ] ; then
|
||||
echo "Usage: $0 mod_pagespeed_dir nginx_executable"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
MOD_PAGESPEED_DIR="$1"
|
||||
NGINX_EXECUTABLE="$2"
|
||||
|
||||
: ${PRIMARY_PORT:=8050}
|
||||
: ${SECONDARY_PORT:=8051}
|
||||
: ${CONTROLLER_PORT:=8053}
|
||||
: ${RCPORT:=9991}
|
||||
: ${PAGESPEED_TEST_HOST:=selfsigned.modpagespeed.com}
|
||||
|
||||
this_dir="$( cd $(dirname "$0") && pwd)"
|
||||
|
||||
function run_test_checking_failure() {
|
||||
USE_VALGRIND="$USE_VALGRIND" \
|
||||
PRIMARY_PORT="$PRIMARY_PORT" \
|
||||
SECONDARY_PORT="$SECONDARY_PORT" \
|
||||
MOD_PAGESPEED_DIR="$MOD_PAGESPEED_DIR" \
|
||||
NGINX_EXECUTABLE="$NGINX_EXECUTABLE" \
|
||||
PAGESPEED_TEST_HOST="$PAGESPEED_TEST_HOST" \
|
||||
RUN_TESTS="$RUN_TESTS" \
|
||||
CONTROLLER_PORT="$CONTROLLER_PORT" \
|
||||
RCPORT="$RCPORT" \
|
||||
bash "$this_dir/nginx_system_test.sh"
|
||||
STATUS=$?
|
||||
echo "With $@ setup."
|
||||
case $STATUS in
|
||||
0)
|
||||
return # No failure.
|
||||
;;
|
||||
3)
|
||||
return # Only expected failures.
|
||||
;;
|
||||
4)
|
||||
return # Return passing error code when running manually.
|
||||
;;
|
||||
*)
|
||||
exit 1 # Real failure.
|
||||
esac
|
||||
}
|
||||
|
||||
if $TEST_SERF_FETCHER; then
|
||||
NATIVE_FETCHER=off run_test_checking_failure "serf fetcher"
|
||||
fi
|
||||
|
||||
if $TEST_NATIVE_FETCHER; then
|
||||
NATIVE_FETCHER=on run_test_checking_failure "native fetcher"
|
||||
fi
|
||||
193
ngx_pagespeed-latest-beta/test/valgrind.sup
Normal file
193
ngx_pagespeed-latest-beta/test/valgrind.sup
Normal file
@@ -0,0 +1,193 @@
|
||||
# Copyright 2013 Google Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
# Author: oschaaf@we-amp.com (Otto van der Schaaf)
|
||||
|
||||
|
||||
# The first few suppressions can be found in other modules
|
||||
# and easily found when searched for, and seem false positives.
|
||||
{
|
||||
<nginx false positive>
|
||||
Memcheck:Param
|
||||
socketcall.sendmsg(msg.msg_iov[i])
|
||||
fun:__sendmsg_nocancel
|
||||
fun:ngx_write_channel
|
||||
fun:ngx_signal_worker_processes
|
||||
fun:ngx_master_process_cycle
|
||||
fun:main
|
||||
}
|
||||
{
|
||||
<nginx false positive>
|
||||
Memcheck:Param
|
||||
socketcall.sendmsg(msg.msg_iov[i])
|
||||
fun:__sendmsg_nocancel
|
||||
fun:ngx_write_channel
|
||||
fun:ngx_master_process_cycle
|
||||
fun:main
|
||||
}
|
||||
{
|
||||
<nginx false positive>
|
||||
Memcheck:Param
|
||||
socketcall.sendmsg(msg.msg_iov[i])
|
||||
fun:__sendmsg_nocancel
|
||||
fun:ngx_write_channel
|
||||
fun:ngx_pass_open_channel
|
||||
fun:ngx_start_cache_manager_processes
|
||||
fun:ngx_master_process_cycle
|
||||
fun:main
|
||||
}
|
||||
{
|
||||
<nginx false positive>
|
||||
Memcheck:Param
|
||||
socketcall.sendmsg(msg.msg_iov[i])
|
||||
fun:__sendmsg_nocancel
|
||||
fun:ngx_write_channel
|
||||
fun:ngx_pass_open_channel
|
||||
fun:ngx_start_cache_manager_processes
|
||||
fun:ngx_master_process_cycle
|
||||
fun:main
|
||||
}
|
||||
{
|
||||
<nginx false positive>
|
||||
Memcheck:Leak
|
||||
fun:malloc
|
||||
fun:ngx_alloc
|
||||
fun:ngx_event_process_init
|
||||
fun:ngx_worker_process_init
|
||||
fun:ngx_worker_process_cycle
|
||||
fun:ngx_spawn_process
|
||||
fun:ngx_start_worker_processes
|
||||
fun:ngx_master_process_cycle
|
||||
fun:main
|
||||
}
|
||||
{
|
||||
<nginx false positive>
|
||||
Memcheck:Param
|
||||
socketcall.sendmsg(msg.msg_iov[i])
|
||||
fun:__sendmsg_nocancel
|
||||
fun:ngx_write_channel
|
||||
fun:ngx_pass_open_channel
|
||||
fun:ngx_start_worker_processes
|
||||
fun:ngx_master_process_cycle
|
||||
fun:main
|
||||
}
|
||||
|
||||
# similar to http://trac.nginx.org/nginx/ticket/369
|
||||
{
|
||||
<nginx false positive>
|
||||
Memcheck:Param
|
||||
pwrite64(buf)
|
||||
obj:/lib/x86_64-linux-gnu/libpthread-2.15.so
|
||||
fun:ngx_write_file
|
||||
fun:ngx_write_chain_to_file
|
||||
fun:ngx_write_chain_to_temp_file
|
||||
fun:ngx_event_pipe_write_chain_to_temp_file
|
||||
fun:ngx_event_pipe
|
||||
fun:ngx_http_upstream_process_upstream
|
||||
fun:ngx_http_upstream_process_header
|
||||
fun:ngx_http_upstream_handler
|
||||
fun:ngx_epoll_process_events
|
||||
fun:ngx_process_events_and_timers
|
||||
fun:ngx_worker_process_cycle
|
||||
}
|
||||
# Mentioned in https://github.com/pagespeed/ngx_pagespeed/issues/103
|
||||
# Assuming a false postives as the issue is closed.
|
||||
{
|
||||
<nginx false positive>
|
||||
Memcheck:Param
|
||||
write(buf)
|
||||
obj:/lib/x86_64-linux-gnu/libpthread-2.15.so
|
||||
fun:ngx_log_error_core
|
||||
fun:ngx_http_parse_complex_uri
|
||||
fun:ngx_http_process_request_uri
|
||||
fun:ngx_http_process_request_line
|
||||
fun:ngx_http_wait_request_handler
|
||||
fun:ngx_epoll_process_events
|
||||
fun:ngx_process_events_and_timers
|
||||
fun:ngx_worker_process_cycle
|
||||
fun:ngx_spawn_process
|
||||
fun:ngx_start_worker_processes
|
||||
fun:ngx_master_process_cycle
|
||||
}
|
||||
|
||||
# Extra suppresions for testing in release mode:
|
||||
|
||||
{
|
||||
<re2 uninitialised value in optimized code>
|
||||
Memcheck:Cond
|
||||
fun:_ZN3re24Prog8OptimizeEv
|
||||
...
|
||||
}
|
||||
{
|
||||
<re2 uninitialised value in optimized code>
|
||||
Memcheck:Value8
|
||||
fun:_ZN3re24Prog8OptimizeEv
|
||||
...
|
||||
}
|
||||
|
||||
{
|
||||
<re2 uninitialised value in optimized code>
|
||||
Memcheck:Cond
|
||||
fun:_ZN3re2L4AddQEPNS_9SparseSetEi
|
||||
...
|
||||
}
|
||||
|
||||
{
|
||||
<re2 uninitialised value in optimized code>
|
||||
Memcheck:Value8
|
||||
fun:_ZN3re2L4AddQEPNS_9SparseSetEi
|
||||
...
|
||||
}
|
||||
|
||||
{
|
||||
<re2 uninitialized value in optimized code>
|
||||
Memcheck:Value8
|
||||
fun:_ZN3re23DFA10AddToQueueEPNS0_5WorkqEij
|
||||
...
|
||||
}
|
||||
|
||||
{
|
||||
<re2 uninitialized value in optimized code>
|
||||
Memcheck:Cond
|
||||
fun:_ZN3re23DFA10AddToQueueEPNS0_5WorkqEij
|
||||
...
|
||||
}
|
||||
|
||||
{
|
||||
<re2 uninitialized value in optimized code>
|
||||
Memcheck:Value8
|
||||
fun:_ZNK3re210SparseSetTIvE8containsEi
|
||||
...
|
||||
}
|
||||
|
||||
{
|
||||
<re2 uninitialized value in optimized code>
|
||||
Memcheck:Cond
|
||||
fun:_ZNK3re210SparseSetTIvE8containsEi
|
||||
...
|
||||
}
|
||||
|
||||
{
|
||||
<re2 uninitialized value in optimized code>
|
||||
Memcheck:Value8
|
||||
fun:_ZNK3re211SparseArrayIiE9has_indexEi
|
||||
...
|
||||
}
|
||||
|
||||
{
|
||||
<re2 uninitialized value in optimized code>
|
||||
Memcheck:Cond
|
||||
fun:_ZNK3re211SparseArrayIiE9has_indexEi
|
||||
...
|
||||
}
|
||||
Reference in New Issue
Block a user