Initial Commit

ngx_pagespeed-latest-beta/src/log_message_handler.cc

@@ -0,0 +1,114 @@
+// Copyright 2010 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: jmarantz@google.com (Joshua Marantz)
+// Author: sligocki@google.com (Shawn Ligocki)
+// Author: jefftk@google.com (Jeff Kaufman)
+
+// TODO(jefftk): share more of this code with apache's log_message_handler
+
+#include "log_message_handler.h"
+
+#include <unistd.h>
+
+#include <limits>
+#include <string>
+
+#include "base/debug/debugger.h"
+#include "base/debug/stack_trace.h"
+#include "base/logging.h"
+#include "net/instaweb/public/version.h"
+#include "pagespeed/kernel/base/string_util.h"
+
+// Make sure we don't attempt to use LOG macros here, since doing so
+// would cause us to go into an infinite log loop.
+#undef LOG
+#define LOG USING_LOG_HERE_WOULD_CAUSE_INFINITE_RECURSION
+
+namespace {
+
+ngx_log_t* ngx_log = NULL;
+
+ngx_uint_t GetNgxLogLevel(int severity) {
+  switch (severity) {
+    case logging::LOG_INFO:
+      return NGX_LOG_INFO;
+    case logging::LOG_WARNING:
+      return NGX_LOG_WARN;
+    case logging::LOG_ERROR:
+      return NGX_LOG_ERR;
+    case logging::LOG_ERROR_REPORT:
+    case logging::LOG_FATAL:
+      return NGX_LOG_ALERT;
+    default:  // For VLOG(s)
+      return NGX_LOG_DEBUG;
+  }
+}
+
+bool LogMessageHandler(int severity, const char* file, int line,
+                       size_t message_start, const GoogleString& str) {
+  ngx_uint_t this_log_level = GetNgxLogLevel(severity);
+
+  GoogleString message = str;
+  if (severity == logging::LOG_FATAL) {
+    if (base::debug::BeingDebugged()) {
+      base::debug::BreakDebugger();
+    } else {
+      base::debug::StackTrace trace;
+      std::ostringstream stream;
+      trace.OutputToStream(&stream);
+      message.append(stream.str());
+    }
+  }
+
+  // Trim the newline off the end of the message string.
+  size_t last_msg_character_index = message.length() - 1;
+  if (message[last_msg_character_index] == '\n') {
+    message.resize(last_msg_character_index);
+  }
+
+  ngx_log_error(this_log_level, ngx_log, 0, "[ngx_pagespeed %s] %s",
+                net_instaweb::kModPagespeedVersion,
+                message.c_str());
+
+  if (severity == logging::LOG_FATAL) {
+    // Crash the process to generate a dump.
+    base::debug::BreakDebugger();
+  }
+
+  return true;
+}
+
+}  // namespace
+
+
+namespace net_instaweb {
+
+namespace log_message_handler {
+
+
+void Install(ngx_log_t* log_in) {
+  ngx_log = log_in;
+  logging::SetLogMessageHandler(&LogMessageHandler);
+
+  // All VLOG(2) and higher will be displayed as DEBUG logs if the nginx log
+  // level is DEBUG.
+  if (ngx_log->log_level >= NGX_LOG_DEBUG) {
+    logging::SetMinLogLevel(-2);
+  }
+}
+
+}  // namespace log_message_handler
+
+}  // namespace net_instaweb

ngx_pagespeed-latest-beta/src/log_message_handler.h

@@ -0,0 +1,44 @@
+// Copyright 2010 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: jmarantz@google.com (Joshua Marantz)
+// Author: sligocki@google.com (Shawn Ligocki)
+// Author: jefftk@google.com (Jeff Kaufman)
+
+#ifndef LOG_MESSAGE_HANDLER_H_
+#define LOG_MESSAGE_HANDLER_H_
+
+extern "C" {
+  #include <ngx_config.h>
+  #include <ngx_core.h>
+  #include <ngx_http.h>
+  #include <ngx_log.h>
+}
+
+namespace net_instaweb {
+
+namespace log_message_handler {
+
+// Install a log message handler that routes LOG() messages to the
+// server error log.  Should be called once at startup.  If server blocks define
+// their own logging files you would expect that LOG() messages would be routed
+// appropriately, but because logging is all done with global variables this
+// isn't possible.
+void Install(ngx_log_t* log_in);
+
+}  // namespace log_message_handler
+
+}  // namespace net_instaweb
+
+#endif  // LOG_MESSAGE_HANDLER_H_

ngx_pagespeed-latest-beta/src/ngx_base_fetch.cc

@@ -0,0 +1,360 @@
+/*
+ * Copyright 2012 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Author: jefftk@google.com (Jeff Kaufman)
+ */
+
+#include "ngx_pagespeed.h"  // Must come first, see comments in CollectHeaders.
+
+#include <unistd.h> //for usleep
+
+#include "ngx_base_fetch.h"
+#include "ngx_event_connection.h"
+#include "ngx_list_iterator.h"
+
+#include "net/instaweb/rewriter/public/rewrite_driver.h"
+#include "net/instaweb/rewriter/public/rewrite_options.h"
+#include "net/instaweb/rewriter/public/rewrite_stats.h"
+#include "pagespeed/kernel/base/google_message_handler.h"
+#include "pagespeed/kernel/base/message_handler.h"
+#include "pagespeed/kernel/base/posix_timer.h"
+#include "pagespeed/kernel/http/response_headers.h"
+
+namespace net_instaweb {
+
+const char kHeadersComplete = 'H';
+const char kFlush = 'F';
+const char kDone = 'D';
+
+NgxEventConnection* NgxBaseFetch::event_connection = NULL;
+int NgxBaseFetch::active_base_fetches = 0;
+
+NgxBaseFetch::NgxBaseFetch(StringPiece url,
+                           ngx_http_request_t* r,
+                           NgxServerContext* server_context,
+                           const RequestContextPtr& request_ctx,
+                           PreserveCachingHeaders preserve_caching_headers,
+                           NgxBaseFetchType base_fetch_type,
+                           const RewriteOptions* options)
+    : AsyncFetch(request_ctx),
+      url_(url.data(), url.size()),
+      request_(r),
+      server_context_(server_context),
+      options_(options),
+      need_flush_(false),
+      done_called_(false),
+      last_buf_sent_(false),
+      references_(2),
+      base_fetch_type_(base_fetch_type),
+      preserve_caching_headers_(preserve_caching_headers),
+      detached_(false),
+      suppress_(false) {
+  if (pthread_mutex_init(&mutex_, NULL)) CHECK(0);
+  __sync_add_and_fetch(&NgxBaseFetch::active_base_fetches, 1);
+}
+
+NgxBaseFetch::~NgxBaseFetch() {
+  pthread_mutex_destroy(&mutex_);
+  __sync_add_and_fetch(&NgxBaseFetch::active_base_fetches, -1);
+}
+
+bool NgxBaseFetch::Initialize(ngx_cycle_t* cycle) {
+  CHECK(event_connection == NULL) << "event connection already set";
+  event_connection = new NgxEventConnection(ReadCallback);
+  return event_connection->Init(cycle);
+}
+
+void NgxBaseFetch::Terminate() {
+  if (event_connection != NULL) {
+    GoogleMessageHandler handler;
+    PosixTimer timer;
+    int64 timeout_us = Timer::kSecondUs * 30;
+    int64 end_us = timer.NowUs() + timeout_us;
+    static unsigned int sleep_microseconds = 100;
+
+    handler.Message(
+        kInfo,"NgxBaseFetch::Terminate rounding up %d active base fetches.",
+        NgxBaseFetch::active_base_fetches);
+
+    // Try to continue processing and get the active base fetch count to 0
+    // untill the timeout expires.
+    // TODO(oschaaf): This needs more work.
+    while (NgxBaseFetch::active_base_fetches > 0 && end_us > timer.NowUs()) {
+      event_connection->Drain();
+      usleep(sleep_microseconds);
+    }
+
+    if (NgxBaseFetch::active_base_fetches != 0) {
+      handler.Message(
+          kWarning,"NgxBaseFetch::Terminate timed out with %d active base fetches.",
+          NgxBaseFetch::active_base_fetches);
+    }
+
+    // Close down the named pipe.
+    event_connection->Shutdown();
+    delete event_connection;
+    event_connection = NULL;
+  }
+}
+
+const char* BaseFetchTypeToCStr(NgxBaseFetchType type) {
+  switch(type) {
+    case kPageSpeedResource:
+      return "ps resource";
+    case kHtmlTransform:
+      return "html transform";
+    case kAdminPage:
+      return "admin page";
+    case kIproLookup:
+      return "ipro lookup";
+    case kPageSpeedProxy:
+      return "pagespeed proxy";
+  }
+  CHECK(false);
+  return "can't get here";
+}
+
+// TODO(oschaaf): replace the ngx_log_error with VLOGS or pass in a
+// MessageHandler and use that.
+void NgxBaseFetch::ReadCallback(const ps_event_data& data) {
+  NgxBaseFetch* base_fetch = reinterpret_cast<NgxBaseFetch*>(data.sender);
+  ngx_http_request_t* r = base_fetch->request();
+  bool detached = base_fetch->detached();
+#if (NGX_DEBUG)  // `type` is unused if NGX_DEBUG isn't set, needed for -Werror.
+  const char* type = BaseFetchTypeToCStr(base_fetch->base_fetch_type_);
+#endif
+  int refcount = base_fetch->DecrementRefCount();
+
+#if (NGX_DEBUG)
+  ngx_log_error(NGX_LOG_DEBUG, ngx_cycle->log, 0,
+     "pagespeed [%p] event: %c. bf:%p (%s) - refcnt:%d - det: %c", r,
+     data.type, base_fetch, type, refcount, detached ? 'Y': 'N');
+#endif
+
+  // If we ended up destructing the base fetch, or the request context is
+  // detached, skip this event.
+  if (refcount == 0 || detached) {
+    return;
+  }
+
+  ps_request_ctx_t* ctx = ps_get_request_context(r);
+
+  // If our request context was zeroed, skip this event.
+  // See https://github.com/pagespeed/ngx_pagespeed/issues/1081
+  if (ctx == NULL) {
+    // Should not happen normally, when it does this message will cause our
+    // system tests to fail.
+    ngx_log_error(NGX_LOG_WARN, ngx_cycle->log, 0,
+        "pagespeed [%p] skipping event: request context gone", r);
+    return;
+  }
+
+  // Normally we expect the sender to equal the active NgxBaseFetch instance.
+  DCHECK(data.sender == ctx->base_fetch);
+
+  // If someone changed our request context or NgxBaseFetch, skip processing.
+  if (data.sender != ctx->base_fetch) {
+      ngx_log_error(NGX_LOG_WARN, ngx_cycle->log, 0,
+          "pagespeed [%p] skipping event: event originating from disassociated"
+          " NgxBaseFetch instance.", r);
+      return;
+  }
+
+  int rc;
+  bool run_posted = true;
+  // If we are unlucky enough to have our connection finalized mid-ipro-lookup,
+  // we must enter a different flow. Also see ps_in_place_check_header_filter().
+  if ((ctx->base_fetch->base_fetch_type_ != kIproLookup)
+      && r->connection->error) {
+    ngx_log_error(NGX_LOG_DEBUG, ngx_cycle->log, 0,
+      "pagespeed [%p] request already finalized %d", r, r->count);
+    rc = NGX_ERROR;
+    run_posted = false;
+  } else {
+    rc = ps_base_fetch::ps_base_fetch_handler(r);
+  }
+
+#if (NGX_DEBUG)
+  ngx_log_error(NGX_LOG_DEBUG, ngx_cycle->log, 0,
+                "pagespeed [%p] ps_base_fetch_handler() returned %d for %c",
+                r, rc, data.type);
+#endif
+
+  ngx_connection_t* c = r->connection;
+  ngx_http_finalize_request(r, rc);
+
+  if (run_posted) {
+    // See http://forum.nginx.org/read.php?2,253006,253061
+    ngx_http_run_posted_requests(c);
+  }
+}
+
+void NgxBaseFetch::Lock() {
+  pthread_mutex_lock(&mutex_);
+}
+
+void NgxBaseFetch::Unlock() {
+  pthread_mutex_unlock(&mutex_);
+}
+
+bool NgxBaseFetch::HandleWrite(const StringPiece& sp,
+                               MessageHandler* handler) {
+  Lock();
+  buffer_.append(sp.data(), sp.size());
+  Unlock();
+  return true;
+}
+
+// should only be called in nginx thread
+ngx_int_t NgxBaseFetch::CopyBufferToNginx(ngx_chain_t** link_ptr) {
+  CHECK(!(done_called_ && last_buf_sent_))
+        << "CopyBufferToNginx() was called after the last buffer was sent";
+
+  // there is no buffer to send
+  if (!done_called_ && buffer_.empty()) {
+    *link_ptr = NULL;
+    return NGX_AGAIN;
+  }
+
+  int rc = string_piece_to_buffer_chain(request_->pool, buffer_, link_ptr,
+                                        done_called_ /* send_last_buf */,
+                                        need_flush_);
+  need_flush_ = false;
+
+  if (rc != NGX_OK) {
+    return rc;
+  }
+
+  // Done with buffer contents now.
+  buffer_.clear();
+
+  if (done_called_) {
+    last_buf_sent_ = true;
+    return NGX_OK;
+  }
+
+  return NGX_AGAIN;
+}
+
+// There may also be a race condition if this is called between the last Write()
+// and Done() such that we're sending an empty buffer with last_buf set, which I
+// think nginx will reject.
+ngx_int_t NgxBaseFetch::CollectAccumulatedWrites(ngx_chain_t** link_ptr) {
+  ngx_int_t rc;
+  Lock();
+  rc = CopyBufferToNginx(link_ptr);
+  Unlock();
+  return rc;
+}
+
+ngx_int_t NgxBaseFetch::CollectHeaders(ngx_http_headers_out_t* headers_out) {
+  // nginx defines _FILE_OFFSET_BITS to 64, which changes the size of off_t.
+  // If a standard header is accidentally included before the nginx header,
+  // on a 32-bit system off_t will be 4 bytes and we don't assign all the
+  // bits of content_length_n. Sanity check that did not happen.
+  // This could use static_assert, but this file is not built with --std=c++11.
+  bool sanity_check_off_t[sizeof(off_t) == 8 ? 1 : -1] __attribute__ ((unused));
+
+  const ResponseHeaders* pagespeed_headers = response_headers();
+
+  if (content_length_known()) {
+     headers_out->content_length = NULL;
+     headers_out->content_length_n = content_length();
+  }
+
+  return copy_response_headers_to_ngx(request_, *pagespeed_headers,
+                                      preserve_caching_headers_);
+}
+
+void NgxBaseFetch::RequestCollection(char type) {
+  if (suppress_) {
+    return;
+  }
+
+  // We must optimistically increment the refcount, and decrement it
+  // when we conclude we failed. If we only increment on a successfull write,
+  // there's a small chance that between writing and adding to the refcount
+  // both pagespeed and nginx will release their refcount -- destructing
+  // this NgxBaseFetch instance.
+  IncrementRefCount();
+  if (!event_connection->WriteEvent(type, this)) {
+    DecrementRefCount();
+  }
+}
+
+void NgxBaseFetch::HandleHeadersComplete() {
+  int status_code = response_headers()->status_code();
+  bool status_ok = (status_code != 0) && (status_code < 400);
+
+  if ((base_fetch_type_ != kIproLookup) || status_ok) {
+    // If this is a 404 response we need to count it in the stats.
+    if (response_headers()->status_code() == HttpStatus::kNotFound) {
+      server_context_->rewrite_stats()->resource_404_count()->Add(1);
+    }
+  }
+
+  RequestCollection(kHeadersComplete);  // Headers available.
+
+  // For the IPRO lookup, supress notification of the nginx side here.
+  // If we send both the headerscomplete event and the one from done, nasty
+  // stuff will happen if we loose the race with with the nginx side destructing
+  // this base fetch instance.
+  if (base_fetch_type_ == kIproLookup && !status_ok) {
+    suppress_ = true;
+  }
+}
+
+bool NgxBaseFetch::HandleFlush(MessageHandler* handler) {
+  Lock();
+  need_flush_ = true;
+  Unlock();
+  RequestCollection(kFlush);  // A new part of the response body is available
+  return true;
+}
+
+int NgxBaseFetch::DecrementRefCount() {
+  return DecrefAndDeleteIfUnreferenced();
+}
+
+int NgxBaseFetch::IncrementRefCount() {
+  return __sync_add_and_fetch(&references_, 1);
+}
+
+int NgxBaseFetch::DecrefAndDeleteIfUnreferenced() {
+  // Creates a full memory barrier.
+  int r = __sync_add_and_fetch(&references_, -1);
+  if (r == 0) {
+    delete this;
+  }
+  return r;
+}
+
+void NgxBaseFetch::HandleDone(bool success) {
+  // TODO(jefftk): it's possible that instead of locking here we can just modify
+  // CopyBufferToNginx to only read done_called_ once.
+  CHECK(!done_called_) << "Done already called!";
+  Lock();
+  done_called_ = true;
+  Unlock();
+  RequestCollection(kDone);
+  DecrefAndDeleteIfUnreferenced();
+}
+
+bool NgxBaseFetch::IsCachedResultValid(const ResponseHeaders& headers) {
+  return OptionsAwareHTTPCacheCallback::IsCacheValid(
+      url_, *options_, request_context(), headers);
+}
+
+}  // namespace net_instaweb

ngx_pagespeed-latest-beta/src/ngx_base_fetch.h

@@ -0,0 +1,192 @@
+/*
+ * Copyright 2012 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+//
+// Collects output from pagespeed and buffers it until nginx asks for it.
+// Notifies nginx via NgxEventConnection to call ReadCallback() when
+// the headers are computed, when a flush should be performed, and when done.
+//
+//  - nginx creates a base fetch and passes it to a new proxy fetch.
+//  - The proxy fetch manages rewriting and thread complexity, and through
+//    several chained steps passes rewritten html to HandleWrite().
+//  - Written data is buffered.
+//  - When HandleHeadersComplete(), HandleFlush(), or HandleDone() is called by
+//    PSOL, events are written to NgxEventConnection which will end up being
+//    handled by ReadCallback() on nginx's thread.
+//    When applicable, request processing will be continued via a call to
+//    ps_base_fetch_handler().
+//  - ps_base_fetch_handler() will pull the header and body bytes from PSOL
+//    via CollectAccumulatedWrites() and write those to the module's output.
+//
+// This class is referred to in three places: the proxy fetch, nginx's request,
+// and pending events written to the associated NgxEventConnection. It must stay
+// alive until the proxy fetch and nginx request are finished, and no more
+// events are pending.
+//  - The proxy fetch will call Done() to indicate this.
+//  - nginx will call Detach() when the associated request is handled
+//    completely (e.g. the request context is about to be destroyed).
+//  - ReadCallback() will call DecrementRefCount() on instances associated to
+//    events it handles.
+//
+// When the last reference is dropped, this class will delete itself.
+//
+// TODO(jmarantz): consider sharing the cache-invalidation infrastructure
+// with ApacheFetch, using a common base class.
+
+#ifndef NGX_BASE_FETCH_H_
+#define NGX_BASE_FETCH_H_
+
+extern "C" {
+#include <ngx_http.h>
+}
+
+#include <pthread.h>
+
+#include "ngx_pagespeed.h"
+
+#include "ngx_event_connection.h"
+#include "ngx_server_context.h"
+
+#include "net/instaweb/http/public/async_fetch.h"
+#include "net/instaweb/rewriter/public/rewrite_options.h"
+#include "pagespeed/kernel/base/string.h"
+#include "pagespeed/kernel/http/headers.h"
+
+namespace net_instaweb {
+
+enum NgxBaseFetchType {
+  kIproLookup,
+  kHtmlTransform,
+  kPageSpeedResource,
+  kAdminPage,
+  kPageSpeedProxy
+};
+
+class NgxBaseFetch : public AsyncFetch {
+ public:
+  NgxBaseFetch(StringPiece url, ngx_http_request_t* r,
+               NgxServerContext* server_context,
+               const RequestContextPtr& request_ctx,
+               PreserveCachingHeaders preserve_caching_headers,
+               NgxBaseFetchType base_fetch_type,
+               const RewriteOptions* options);
+  virtual ~NgxBaseFetch();
+
+  // Statically initializes event_connection, require for PSOL and nginx to
+  // communicate.
+  static bool Initialize(ngx_cycle_t* cycle);
+
+  // Attempts to finish up request processing queued up in the named pipe and
+  // PSOL for a fixed amount of time. If time is up, a fast and rough shutdown
+  // is attempted.
+  // Statically terminates and NULLS event_connection.
+  static void Terminate();
+
+  static void ReadCallback(const ps_event_data& data);
+
+  // Puts a chain in link_ptr if we have any output data buffered.  Returns
+  // NGX_OK on success, NGX_ERROR on errors.  If there's no data to send, sends
+  // data only if Done() has been called.  Indicates the end of output by
+  // setting last_buf on the last buffer in the chain.
+  //
+  // Sets link_ptr to a chain of as many buffers are needed for the output.
+  //
+  // Called by nginx in response to seeing a byte on the pipe.
+  ngx_int_t CollectAccumulatedWrites(ngx_chain_t** link_ptr);
+
+  // Copies response headers into headers_out.
+  //
+  // Called by nginx before calling CollectAccumulatedWrites() for the first
+  // time for resource fetches.  Not called at all for proxy fetches.
+  ngx_int_t CollectHeaders(ngx_http_headers_out_t* headers_out);
+
+  // Called by nginx to decrement the refcount.
+  int DecrementRefCount();
+
+  // Called by pagespeed to increment the refcount.
+  int IncrementRefCount();
+
+  // Detach() is called when the nginx side releases this base fetch. It
+  // sets detached_ to true and decrements the refcount. We need to know
+  // this to be able to handle events which nginx request context has been
+  // released while the event was in-flight.
+  void Detach() { detached_ = true; DecrementRefCount(); }
+
+  bool detached() { return detached_; }
+
+  ngx_http_request_t* request() { return request_; }
+  NgxBaseFetchType base_fetch_type() { return base_fetch_type_; }
+
+  bool IsCachedResultValid(const ResponseHeaders& headers) override;
+
+ private:
+  virtual bool HandleWrite(const StringPiece& sp, MessageHandler* handler);
+  virtual bool HandleFlush(MessageHandler* handler);
+  virtual void HandleHeadersComplete();
+  virtual void HandleDone(bool success);
+
+  // Indicate to nginx that we would like it to call
+  // CollectAccumulatedWrites().
+  void RequestCollection(char type);
+
+  // Lock must be acquired first.
+  // Returns:
+  //   NGX_ERROR: failure
+  //   NGX_AGAIN: still has buffer to send, need to checkout link_ptr
+  //   NGX_OK: done, HandleDone has been called
+  // Allocates an nginx buffer, copies our buffer_ contents into it, clears
+  // buffer_.
+  ngx_int_t CopyBufferToNginx(ngx_chain_t** link_ptr);
+
+  void Lock();
+  void Unlock();
+
+  // Called by Done() and Release().  Decrements our reference count, and if
+  // it's zero we delete ourself.
+  int DecrefAndDeleteIfUnreferenced();
+
+  static NgxEventConnection* event_connection;
+
+  // Live count of NgxBaseFetch instances that are currently in use.
+  static int active_base_fetches;
+
+  GoogleString url_;
+  ngx_http_request_t* request_;
+  GoogleString buffer_;
+  NgxServerContext* server_context_;
+  const RewriteOptions* options_;
+  bool need_flush_;
+  bool done_called_;
+  bool last_buf_sent_;
+  // How many active references there are to this fetch. Starts at two,
+  // decremented once when Done() is called and once when Detach() is called.
+  // Incremented for each event written by pagespeed for this NgxBaseFetch, and
+  // decremented on the nginx side for each event read for it.
+  int references_;
+  pthread_mutex_t mutex_;
+  NgxBaseFetchType base_fetch_type_;
+  PreserveCachingHeaders preserve_caching_headers_;
+  // Set to true just before the nginx side releases its reference
+  bool detached_;
+  bool suppress_;
+
+  DISALLOW_COPY_AND_ASSIGN(NgxBaseFetch);
+};
+
+}  // namespace net_instaweb
+
+#endif  // NGX_BASE_FETCH_H_

ngx_pagespeed-latest-beta/src/ngx_caching_headers.cc

@@ -0,0 +1,51 @@
+/*
+ * Copyright 2013 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+
+#include "ngx_caching_headers.h"
+#include "ngx_list_iterator.h"
+
+#include "ngx_pagespeed.h"
+
+namespace net_instaweb {
+
+bool NgxCachingHeaders::Lookup(const StringPiece& key,
+                               StringPieceVector* values) {
+  ngx_table_elt_t* header;
+  NgxListIterator it(&(request_->headers_out.headers.part));
+  while ((header = it.Next()) != NULL) {
+    if (header->hash != 0 && key == str_to_string_piece(header->key)) {
+      // This will be called multiple times if there are multiple headers with
+      // this name.  Each time it will append to values.
+      SplitStringPieceToVector(str_to_string_piece(header->value), ",", values,
+                               true /* omit empty strings */);
+    }
+  }
+
+  if (values->size() == 0) {
+    // No header found with this name.
+    return false;
+  }
+
+  for (int i = 0, n = values->size(); i < n; ++i) {
+    TrimWhitespace(&((*values)[i]));
+  }
+
+  return true;
+}
+
+}  // namespace net_instaweb

ngx_pagespeed-latest-beta/src/ngx_caching_headers.h

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2013 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+
+#ifndef NGX_CACHING_HEADERS_H_
+#define NGX_CACHING_HEADERS_H_
+
+extern "C" {
+  #include <ngx_http.h>
+}
+
+#include "pagespeed/kernel/base/basictypes.h"
+#include "pagespeed/kernel/http/caching_headers.h"
+
+namespace net_instaweb {
+
+// Based off of ApacheCachingHeaders in net/instaweb/apache/header_util.cc
+// Needed so ps_header_filter can call GenerateDisabledCacheControl().
+class NgxCachingHeaders : public CachingHeaders {
+ public:
+  explicit NgxCachingHeaders(ngx_http_request_t* request)
+      : CachingHeaders(request->headers_out.status),
+        request_(request) {
+  }
+
+  virtual bool Lookup(const StringPiece& key, StringPieceVector* values);
+
+  virtual bool IsLikelyStaticResourceType() const {
+    DCHECK(false);  // not called in our use-case.
+    return false;
+  }
+
+  virtual bool IsCacheableResourceStatusCode() const {
+    DCHECK(false);  // not called in our use-case.
+    return false;
+  }
+
+ private:
+  ngx_http_request_t* request_;
+
+  DISALLOW_COPY_AND_ASSIGN(NgxCachingHeaders);
+};
+
+}  // namespace net_instaweb
+
+#endif  // NGX_CACHING_HEADERS_H_

ngx_pagespeed-latest-beta/src/ngx_event_connection.cc

@@ -0,0 +1,172 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+
+extern "C" {
+
+#include <ngx_channel.h>
+
+}
+
+#include "ngx_event_connection.h"
+
+#include "pagespeed/kernel/base/google_message_handler.h"
+#include "pagespeed/kernel/base/message_handler.h"
+
+namespace net_instaweb {
+
+  NgxEventConnection::NgxEventConnection(callbackPtr callback)
+    : event_handler_(callback) {
+}
+
+bool NgxEventConnection::Init(ngx_cycle_t* cycle) {
+  int file_descriptors[2];
+
+  if (pipe(file_descriptors) != 0) {
+    ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, "pagespeed: pipe() failed");
+    return false;
+  }
+  if (ngx_nonblocking(file_descriptors[0]) == -1) {
+    ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno,
+                  ngx_nonblocking_n "pagespeed:  pipe[0] failed");
+  } else if (ngx_nonblocking(file_descriptors[1]) == -1) {
+    ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno,
+                  ngx_nonblocking_n "pagespeed:  pipe[1] failed");
+  } else if (!CreateNgxConnection(cycle, file_descriptors[0])) {
+    ngx_log_error(NGX_LOG_EMERG, cycle->log, 0,
+                  "pagespeed: failed to create connection.");
+  } else {
+    pipe_read_fd_ = file_descriptors[0];
+    pipe_write_fd_ = file_descriptors[1];
+    return true;
+  }
+  close(file_descriptors[0]);
+  close(file_descriptors[1]);
+  return false;
+}
+
+bool NgxEventConnection::CreateNgxConnection(ngx_cycle_t* cycle,
+                                             ngx_fd_t pipe_fd) {
+  // pipe_fd (the read side of the pipe will end up as c->fd on the
+  // underlying ngx_connection_t that gets created here)
+  ngx_int_t rc = ngx_add_channel_event(cycle, pipe_fd, NGX_READ_EVENT,
+      &NgxEventConnection::ReadEventHandler);
+  return rc  == NGX_OK;
+}
+
+void NgxEventConnection::ReadEventHandler(ngx_event_t* ev) {
+  ngx_connection_t* c = static_cast<ngx_connection_t*>(ev->data);
+  ngx_int_t result = ngx_handle_read_event(ev, 0);
+  if (result != NGX_OK) {
+    CHECK(false) << "pagespeed: ngx_handle_read_event error: " << result;
+  }
+
+  if (ev->timedout) {
+    ev->timedout = 0;
+    return;
+  }
+
+  if (!NgxEventConnection::ReadAndNotify(c->fd)) {
+    // This was copied from ngx_channel_handler(): for epoll, we need to call
+    // ngx_del_conn(). Sadly, no documentation as to why.
+    if (ngx_event_flags & NGX_USE_EPOLL_EVENT) {
+      ngx_del_conn(c, 0);
+    }
+    ngx_close_connection(c);
+    ngx_del_event(ev, NGX_READ_EVENT, 0);
+  }
+}
+
+// Deserialize ps_event_data's from the pipe as they become available.
+// Subsequently do some bookkeeping, cleanup, and error checking to keep
+// the mess out of ps_base_fetch_handler.
+bool NgxEventConnection::ReadAndNotify(ngx_fd_t fd) {
+  while (true) {
+    // We read only one ps_event_data at a time for now:
+    // We can end up recursing all the way and end up calling ourselves here.
+    // If that happens in the middle of looping over multiple ps_event_data's we
+    // have obtained with read(), the results from the next read() will make us
+    // process events out of order. Which can give headaches.
+    // Alternatively, we could maintain a queue to make sure we process in
+    // sequence
+    ps_event_data data;
+    ngx_int_t size = read(fd, static_cast<void*>(&data), sizeof(data));
+
+    if (size == -1) {
+      if (errno == EINTR) {
+        continue;
+      // TODO(oschaaf): should we worry about spinning here?
+      } else if (ngx_errno == EAGAIN || ngx_errno == EWOULDBLOCK) {
+        return true;
+      }
+    }
+
+    if (size <= 0) {
+      return false;
+    }
+
+    data.connection->event_handler_(data);
+    return true;
+  }
+}
+
+bool NgxEventConnection::WriteEvent(void* sender) {
+  return WriteEvent('X' /* Anything char is fine */, sender);
+}
+
+bool NgxEventConnection::WriteEvent(char type, void* sender) {
+  ssize_t size = 0;
+  ps_event_data data;
+
+  ngx_memzero(&data, sizeof(data));
+  data.type = type;
+  data.sender = sender;
+  data.connection = this;
+
+  while (true) {
+    size = write(pipe_write_fd_,
+                 static_cast<void*>(&data), sizeof(data));
+    if (size == sizeof(data)) {
+      return true;
+    } else if (size == -1) {
+      // TODO(oschaaf): should we worry about spinning here?
+      if (ngx_errno == EINTR || ngx_errno == EAGAIN
+          || ngx_errno == EWOULDBLOCK) {
+        continue;
+      } else {
+        return false;
+      }
+    } else {
+      CHECK(false) << "pagespeed: unexpected return value from write(): "
+                   << size;
+    }
+  }
+  CHECK(false) << "Should not get here";
+  return false;
+}
+
+// Reads and processes what is available in the pipe.
+void NgxEventConnection::Drain() {
+  NgxEventConnection::ReadAndNotify(pipe_read_fd_);
+}
+
+void NgxEventConnection::Shutdown() {
+  close(pipe_write_fd_);
+  close(pipe_read_fd_);
+}
+
+}  // namespace net_instaweb

ngx_pagespeed-latest-beta/src/ngx_event_connection.h

@@ -0,0 +1,84 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+//
+// NgxEventConnection implements a means to send events from other threads to
+// nginx's event loop, and is implemented by a named pipe under the hood.
+// A single instance is used by NgxBaseFetch, and one instance is created per
+// NgxUrlAsyncFetcher when native fetching is on.
+
+#ifndef NGX_EVENT_CONNECTION_H_
+#define NGX_EVENT_CONNECTION_H_
+
+extern "C" {
+#include <ngx_http.h>
+}
+
+#include <pthread.h>
+
+#include "pagespeed/kernel/base/string.h"
+#include "pagespeed/kernel/http/headers.h"
+
+namespace net_instaweb {
+
+class NgxEventConnection;
+
+// Represents a single event that can be written to or read from the pipe.
+// Technically, sender is the only data we need to send. type and connection are
+// included to provide a means to trace the events along with some more
+// info.
+typedef struct {
+  char type;
+  void* sender;
+  NgxEventConnection* connection;
+} ps_event_data;
+
+// Handler signature for receiving events
+typedef void (*callbackPtr)(const ps_event_data&);
+
+// Abstracts a connection to nginx through which events can be written.
+class NgxEventConnection {
+ public:
+  explicit NgxEventConnection(callbackPtr handler);
+
+  // Creates the file descriptors and ngx_connection_t required for event
+  // messaging between pagespeed and nginx.
+  bool Init(ngx_cycle_t* cycle);
+  // Shuts down the underlying file descriptors and connection created in Init()
+  void Shutdown();
+  // Constructs a ps_event_data and writes it to the underlying named pipe.
+  bool WriteEvent(char type, void* sender);
+  // Convenience overload for clients that have a single event type.
+  bool WriteEvent(void* sender);
+  // Reads and processes what is available in the named pipe's buffer.
+  void Drain();
+ private:
+  static bool CreateNgxConnection(ngx_cycle_t* cycle, ngx_fd_t pipe_fd);
+  static void ReadEventHandler(ngx_event_t* e);
+  static bool ReadAndNotify(ngx_fd_t fd);
+
+  callbackPtr event_handler_;
+  // We own these file descriptors
+  ngx_fd_t pipe_write_fd_;
+  ngx_fd_t pipe_read_fd_;
+
+  DISALLOW_COPY_AND_ASSIGN(NgxEventConnection);
+};
+
+}  // namespace net_instaweb
+
+#endif  // NGX_EVENT_CONNECTION_H_

ngx_pagespeed-latest-beta/src/ngx_fetch.cc

@@ -0,0 +1,951 @@
+/*
+ * Copyright 2012 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: x.dinic@gmail.com (Junmin Xiong)
+//
+//  - The fetch is started by the main thread.
+//  - Resolver event was hooked when a NgxFetch starting. It could
+//    lookup the IP of the domain asynchronously from the DNS server.
+//  - When NgxFetchResolveDone is called, It will create the request and the
+//    connection. Add the write and read event to the epoll structure.
+//  - The read handler parses the response. Add the response to the buffer at
+//    last.
+
+// TODO(oschaaf): Currently the first applicable connection is picked from the
+// pool when re-using connections. Perhaps it would be worth it to pick the one
+// that was active the longest time ago to keep a larger pool available.
+// TODO(oschaaf): style: reindent namespace according to google C++ style guide
+// TODO(oschaaf): Retry mechanism for failures on a re-used k-a connection.
+// Currently we don't think it's going to be an issue, see the comments at
+// https://github.com/pagespeed/ngx_pagespeed/pull/781.
+
+extern "C" {
+#include <nginx.h>
+}
+
+#include "ngx_fetch.h"
+
+#include "base/logging.h"
+
+#include <algorithm>
+#include <string>
+#include <typeinfo>
+#include <vector>
+
+#include "net/instaweb/http/public/async_fetch.h"
+#include "net/instaweb/http/public/inflating_fetch.h"
+#include "net/instaweb/public/version.h"
+#include "net/instaweb/public/global_constants.h"
+#include "pagespeed/kernel/base/basictypes.h"
+#include "pagespeed/kernel/base/condvar.h"
+#include "pagespeed/kernel/base/message_handler.h"
+#include "pagespeed/kernel/base/pool.h"
+#include "pagespeed/kernel/base/pool_element.h"
+#include "pagespeed/kernel/base/scoped_ptr.h"
+#include "pagespeed/kernel/base/statistics.h"
+#include "pagespeed/kernel/base/string_writer.h"
+#include "pagespeed/kernel/base/string_util.h"
+#include "pagespeed/kernel/base/thread_system.h"
+#include "pagespeed/kernel/base/timer.h"
+#include "pagespeed/kernel/base/writer.h"
+#include "pagespeed/kernel/http/request_headers.h"
+#include "pagespeed/kernel/http/response_headers.h"
+#include "pagespeed/kernel/http/response_headers_parser.h"
+
+namespace net_instaweb {
+
+NgxConnection::NgxConnectionPool NgxConnection::connection_pool;
+PthreadMutex NgxConnection::connection_pool_mutex;
+// Default keepalive 60s.
+const int64 NgxConnection::keepalive_timeout_ms = 60000;
+const GoogleString NgxConnection::ka_header =
+    StrCat("keep-alive ",
+           Integer64ToString(NgxConnection::keepalive_timeout_ms));
+
+NgxConnection::NgxConnection(MessageHandler* handler,
+                             int max_keepalive_requests) {
+  c_ = NULL;
+  max_keepalive_requests_ = max_keepalive_requests;
+  handler_ = handler;
+  // max_keepalive_requests specifies the number of http requests that are
+  // allowed to be performed over a single connection. So, a
+  // max_keepalive_requests of 1 effectively disables keepalive.
+  keepalive_ = max_keepalive_requests_ > 1;
+}
+
+NgxConnection::~NgxConnection() {
+  CHECK(c_ == NULL) << "NgxConnection: Underlying connection should be NULL";
+}
+
+void NgxConnection::Terminate() {
+  for (NgxConnectionPool::iterator p = connection_pool.begin();
+       p != connection_pool.end(); ++p) {
+    NgxConnection* nc = *p;
+    ngx_close_connection(nc->c_);
+    nc->c_ = NULL;
+    delete nc;
+  }
+  connection_pool.Clear();
+}
+
+NgxConnection* NgxConnection::Connect(ngx_peer_connection_t* pc,
+                                      MessageHandler* handler,
+                                      int max_keepalive_requests) {
+  NgxConnection* nc;
+  {
+    ScopedMutex lock(&NgxConnection::connection_pool_mutex);
+
+    for (NgxConnectionPool::iterator p = connection_pool.begin();
+         p != connection_pool.end(); ++p) {
+      nc = *p;
+
+      if (ngx_memn2cmp(static_cast<u_char*>(nc->sockaddr_),
+                       reinterpret_cast<u_char*>(pc->sockaddr),
+                       nc->socklen_, pc->socklen) == 0) {
+        CHECK(nc->c_->idle) << "Pool should only contain idle connections!";
+
+        nc->c_->idle = 0;
+        nc->c_->log = pc->log;
+        nc->c_->read->log = pc->log;
+        nc->c_->write->log = pc->log;
+        if (nc->c_->pool != NULL) {
+          nc->c_->pool->log = pc->log;
+        }
+
+        if (nc->c_->read->timer_set) {
+          ngx_del_timer(nc->c_->read);
+        }
+        connection_pool.Remove(nc);
+
+        ngx_log_error(NGX_LOG_DEBUG, pc->log, 0,
+                      "NgxFetch: re-using connection %p (pool size: %l)",
+                      nc, connection_pool.size());
+        return nc;
+      }
+    }
+  }
+
+  int rc = ngx_event_connect_peer(pc);
+  if (rc == NGX_ERROR || rc == NGX_DECLINED || rc == NGX_BUSY) {
+    return NULL;
+  }
+
+  // NgxConnection deletes itself if NgxConnection::Close()
+  nc = new NgxConnection(handler, max_keepalive_requests);
+  nc->SetSock(reinterpret_cast<u_char*>(pc->sockaddr), pc->socklen);
+  nc->c_ = pc->connection;
+  return nc;
+}
+
+void NgxConnection::Close() {
+  bool removed_from_pool = false;
+
+  {
+    ScopedMutex lock(&NgxConnection::connection_pool_mutex);
+    for (NgxConnectionPool::iterator p = connection_pool.begin();
+         p != connection_pool.end(); ++p) {
+      if (*p == this) {
+        // When we get here, that means that the connection either has timed
+        // out or has been closed remotely.
+        connection_pool.Remove(this);
+        ngx_log_error(NGX_LOG_DEBUG, c_->log, 0,
+                      "NgxFetch: removed connection %p (pool size: %l)",
+                      this, connection_pool.size());
+        removed_from_pool = true;
+        break;
+      }
+    }
+  }
+
+  max_keepalive_requests_--;
+
+  if (c_->read->timer_set) {
+    ngx_del_timer(c_->read);
+  }
+
+  if (c_->write->timer_set) {
+    ngx_del_timer(c_->write);
+  }
+
+  if (!keepalive_ || max_keepalive_requests_ <= 0 || removed_from_pool) {
+    ngx_close_connection(c_);
+    c_ = NULL;
+    delete this;
+    return;
+  }
+
+  ngx_add_timer(c_->read, static_cast<ngx_msec_t>(
+      NgxConnection::keepalive_timeout_ms));
+
+  c_->data = this;
+  c_->read->handler = NgxConnection::IdleReadHandler;
+  c_->write->handler = NgxConnection::IdleWriteHandler;
+  c_->idle = 1;
+
+  // This connection should not be associated with current fetch.
+  c_->log = ngx_cycle->log;
+  c_->read->log = ngx_cycle->log;
+  c_->write->log = ngx_cycle->log;
+  if (c_->pool != NULL) {
+    c_->pool->log = ngx_cycle->log;
+  }
+
+  // Allow this connection to be re-used, by adding it to the connection pool.
+  {
+    ScopedMutex lock(&NgxConnection::connection_pool_mutex);
+    connection_pool.Add(this);
+    ngx_log_error(NGX_LOG_DEBUG, c_->log, 0,
+                  "NgxFetch: Added connection %p (pool size: %l - "
+                  " max_keepalive_requests_ %d)",
+                  this, connection_pool.size(), max_keepalive_requests_);
+  }
+}
+
+void NgxConnection::IdleWriteHandler(ngx_event_t* ev) {
+  ngx_connection_t* c = static_cast<ngx_connection_t*>(ev->data);
+  u_char buf[1];
+  int n = c->recv(c, buf, 1);
+  if (c->write->timedout) {
+    DCHECK(false) << "NgxFetch: write timeout not expected." << n;
+  }
+  if (n == NGX_AGAIN) {
+    return;
+  }
+}
+
+void NgxConnection::IdleReadHandler(ngx_event_t* ev) {
+  ngx_connection_t* c = static_cast<ngx_connection_t*>(ev->data);
+  NgxConnection* nc = static_cast<NgxConnection*>(c->data);
+
+  if (c->read->timedout) {
+    nc->set_keepalive(false);
+    nc->Close();
+    return;
+  }
+
+  char buf[1];
+  int n;
+
+  // not a timeout event, we should check connection
+  n = recv(c->fd, buf, 1, MSG_PEEK);
+  if (n == -1 && ngx_socket_errno == NGX_EAGAIN) {
+    if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
+      nc->set_keepalive(false);
+      nc->Close();
+      return;
+    }
+
+    return;
+  }
+
+  nc->set_keepalive(false);
+  nc->Close();
+}
+
+NgxFetch::NgxFetch(const GoogleString& url,
+                   AsyncFetch* async_fetch,
+                   MessageHandler* message_handler,
+                   ngx_log_t* log)
+    : str_url_(url),
+      fetcher_(NULL),
+      async_fetch_(async_fetch),
+      parser_(async_fetch->response_headers()),
+      message_handler_(message_handler),
+      bytes_received_(0),
+      fetch_start_ms_(0),
+      fetch_end_ms_(0),
+      done_(false),
+      content_length_(-1),
+      content_length_known_(false),
+      resolver_ctx_(NULL) {
+  ngx_memzero(&url_, sizeof(url_));
+  log_ = log;
+  pool_ = NULL;
+  timeout_event_ = NULL;
+  connection_ = NULL;
+}
+
+NgxFetch::~NgxFetch() {
+  if (timeout_event_ != NULL && timeout_event_->timer_set) {
+    ngx_del_timer(timeout_event_);
+  }
+  if (connection_ != NULL) {
+    connection_->Close();
+    connection_ = NULL;
+  }
+  if (pool_ != NULL) {
+    ngx_destroy_pool(pool_);
+    pool_ = NULL;
+  }
+}
+
+// This function is called by NgxUrlAsyncFetcher::StartFetch.
+bool NgxFetch::Start(NgxUrlAsyncFetcher* fetcher) {
+  fetcher_ = fetcher;
+  bool ok = Init();
+  if (ok) {
+    ngx_log_error(NGX_LOG_DEBUG, log_, 0, "NgxFetch %p: initialized",
+                  this);
+  }  // else Init() will have emitted a reason
+  return ok;
+}
+
+// Create the pool, parse the url, add the timeout event and
+// hook the DNS resolver if needed. Else we connect directly.
+// When this returns false, our caller (NgxUrlAsyncFetcher::StartFetch)
+// will call fetch->CallbackDone()
+bool NgxFetch::Init() {
+  pool_ = ngx_create_pool(12288, log_);
+  if (pool_ == NULL) {
+    message_handler_->Message(kError, "NgxFetch: ngx_create_pool failed");
+    return false;
+  }
+
+  if (!ParseUrl()) {
+    message_handler_->Message(kError, 
+                              "NgxFetch: ParseUrl() failed for [%s]:%s",
+                              str_url_.c_str(), url_.err);
+    return false;
+  }
+
+  timeout_event_ = static_cast<ngx_event_t*>(
+      ngx_pcalloc(pool_, sizeof(ngx_event_t)));
+  if (timeout_event_ == NULL) {
+    message_handler_->Message(kError,
+                              "NgxFetch: ngx_pcalloc failed for timeout");
+    return false;
+  }
+
+  timeout_event_->data = this;
+  timeout_event_->handler = NgxFetch::TimeoutHandler;
+  timeout_event_->log = log_;
+
+  ngx_add_timer(timeout_event_, fetcher_->fetch_timeout_);
+  r_ = static_cast<ngx_http_request_t*>(
+      ngx_pcalloc(pool_, sizeof(ngx_http_request_t)));
+
+  if (r_ == NULL) {
+    message_handler_->Message(kError,
+                              "NgxFetch: ngx_pcalloc failed for timer");
+    return false;
+  }
+  status_ = static_cast<ngx_http_status_t*>(
+      ngx_pcalloc(pool_, sizeof(ngx_http_status_t)));
+
+  if (status_ == NULL) {
+    message_handler_->Message(kError,
+                              "NgxFetch: ngx_pcalloc failed for status");
+    return false;
+  }
+
+  // The host is either a domain name or an IP address.  First check
+  // if it's a valid IP address and only if that fails fall back to
+  // using the DNS resolver.
+
+  // Maybe we have a Proxy.
+  ngx_url_t* tmp_url = &url_;
+  if (fetcher_->proxy_.url.len != 0) {
+    tmp_url = &fetcher_->proxy_;
+  }
+
+  GoogleString s_ipaddress(reinterpret_cast<char*>(tmp_url->host.data),
+                           tmp_url->host.len);
+  ngx_memzero(&sin_, sizeof(sin_));
+  sin_.sin_family = AF_INET;
+  sin_.sin_port = htons(tmp_url->port);
+  sin_.sin_addr.s_addr = inet_addr(s_ipaddress.c_str());
+
+  if (sin_.sin_addr.s_addr == INADDR_NONE) {
+    // inet_addr returned INADDR_NONE, which means the hostname
+    // isn't a valid IP address.  Check DNS.
+    ngx_resolver_ctx_t temp;
+    temp.name.data = tmp_url->host.data;
+    temp.name.len = tmp_url->host.len;
+    resolver_ctx_ = ngx_resolve_start(fetcher_->resolver_, &temp);
+    if (resolver_ctx_ == NULL || resolver_ctx_ == NGX_NO_RESOLVER) {
+      // TODO(oschaaf): this spams the log, but is useful in the fetcher's
+      // current state
+      message_handler_->Message(
+          kError, "NgxFetch: Couldn't start resolving, "
+          "is there a proper resolver configured in nginx.conf?");
+      return false;
+    } else {
+      ngx_log_error(NGX_LOG_DEBUG, log_, 0,
+                    "NgxFetch %p: start resolve for: %s",
+                    this, s_ipaddress.c_str());
+    }
+
+    resolver_ctx_->data = this;
+    resolver_ctx_->name.data = tmp_url->host.data;
+    resolver_ctx_->name.len = tmp_url->host.len;
+
+#if (nginx_version < 1005008)
+    resolver_ctx_->type = NGX_RESOLVE_A;
+#endif
+
+    resolver_ctx_->handler = NgxFetch::ResolveDoneHandler;
+    resolver_ctx_->timeout = fetcher_->resolver_timeout_;
+
+    if (ngx_resolve_name(resolver_ctx_) != NGX_OK) {
+      message_handler_->Message(kWarning,
+                                "NgxFetch: ngx_resolve_name failed");
+      return false;
+    }
+  } else {
+    if (InitRequest() != NGX_OK) {
+      message_handler()->Message(kError, "NgxFetch: InitRequest failed");
+      return false;
+    }
+  }
+  return true;
+}
+
+const char* NgxFetch::str_url() {
+  return str_url_.c_str();
+}
+
+// This function should be called only once. The only argument is sucess or
+// not.
+void NgxFetch::CallbackDone(bool success) {
+  ngx_log_error(NGX_LOG_DEBUG, log_, 0, "NgxFetch %p: CallbackDone: %s",
+                this, success ? "OK":"FAIL");
+
+  if (async_fetch_ == NULL) {
+    LOG(FATAL)
+        << "BUG: NgxFetch callback called more than once on same fetch"
+        << str_url_.c_str() << "(" << this << ").Please report this at"
+        << "https://groups.google.com/forum/#!forum/ngx-pagespeed-discuss";
+    return;
+  }
+
+  release_resolver();
+
+  if (timeout_event_ && timeout_event_->timer_set) {
+    ngx_del_timer(timeout_event_);
+    timeout_event_ = NULL;
+  }
+
+  if (connection_ != NULL) {
+    // Connection will be re-used only on responses that specify
+    // 'Connection: keep-alive' in their headers.
+    bool keepalive = false;
+
+    if (success) {
+      ConstStringStarVector v;
+      if (async_fetch_->response_headers()->Lookup(
+              StringPiece(HttpAttributes::kConnection), &v)) {
+        for (size_t i = 0; i < v.size(); i++) {
+          if (*v[i] == "keep-alive") {
+            keepalive = true;
+            break;
+          } else if (*v[i] == "close") {
+            break;
+          }
+        }
+      }
+      ngx_log_error(NGX_LOG_DEBUG, log_, 0,
+                    "NgxFetch %p: connection %p attempt keep-alive: %s",
+                    this, connection_, keepalive ? "Yes":"No");
+    }
+
+    connection_->set_keepalive(keepalive);
+    connection_->Close();
+    connection_ = NULL;
+  }
+
+  if (fetcher_ != NULL) {
+    if (fetcher_->track_original_content_length()
+        && async_fetch_->response_headers()->Has(
+            HttpAttributes::kXOriginalContentLength)) {
+      async_fetch_->extra_response_headers()->SetOriginalContentLength(
+          bytes_received_);
+    }
+    fetcher_->FetchComplete(this);
+  }
+  async_fetch_->Done(success);
+  async_fetch_ = NULL;
+}
+
+size_t NgxFetch::bytes_received() {
+  return bytes_received_;
+}
+void NgxFetch::bytes_received_add(int64 x) {
+  bytes_received_ += x;
+}
+
+int64 NgxFetch::fetch_start_ms() {
+  return fetch_start_ms_;
+}
+
+void NgxFetch::set_fetch_start_ms(int64 start_ms) {
+  fetch_start_ms_ = start_ms;
+}
+
+int64 NgxFetch::fetch_end_ms() {
+  return fetch_end_ms_;
+}
+
+void NgxFetch::set_fetch_end_ms(int64 end_ms) {
+  fetch_end_ms_ = end_ms;
+}
+
+MessageHandler* NgxFetch::message_handler() {
+  return message_handler_;
+}
+
+bool NgxFetch::ParseUrl() {
+  url_.url.len = str_url_.length();
+  url_.url.data = static_cast<u_char*>(ngx_palloc(pool_, url_.url.len));
+  if (url_.url.data == NULL) {
+    return false;
+  }
+  str_url_.copy(reinterpret_cast<char*>(url_.url.data), str_url_.length(), 0);
+
+  return NgxUrlAsyncFetcher::ParseUrl(&url_, pool_);
+}
+
+// Issue a request after the resolver is done
+void NgxFetch::ResolveDoneHandler(ngx_resolver_ctx_t* resolver_ctx) {
+  NgxFetch* fetch = static_cast<NgxFetch*>(resolver_ctx->data);
+  NgxUrlAsyncFetcher* fetcher = fetch->fetcher_;
+
+  if (resolver_ctx->state != NGX_OK) {
+    if (fetch->timeout_event() != NULL && fetch->timeout_event()->timer_set) {
+      ngx_del_timer(fetch->timeout_event());
+      fetch->set_timeout_event(NULL);
+    }
+    fetch->message_handler()->Message(
+        kWarning, "NgxFetch %p: failed to resolve host [%.*s]", fetch,
+        static_cast<int>(resolver_ctx->name.len), resolver_ctx->name.data);
+    fetch->CallbackDone(false);
+    return;
+  }
+
+  ngx_uint_t i;
+  // Find the first ipv4 address. We don't support ipv6 yet.
+  for (i = 0; i < resolver_ctx->naddrs; i++) {
+    // Old versions of nginx and tengine have a different definition of addrs,
+    // work around to make sure we are using the right type (ngx_addr_t*).
+    ngx_addr_t* ngx_addrs = reinterpret_cast<ngx_addr_t*>(resolver_ctx->addrs);
+    if (typeid(*ngx_addrs) == typeid(*resolver_ctx->addrs)) {
+      if (reinterpret_cast<struct sockaddr_in*>(ngx_addrs[i].sockaddr)
+              ->sin_family == AF_INET) {
+        break;
+      }
+    } else {
+      // We're using an old version that uses in_addr_t* for addrs.
+      break;
+    }
+  }
+
+  // If no suitable ipv4 address was found, we fail.
+  if (i == resolver_ctx->naddrs) {
+    if (fetch->timeout_event() != NULL && fetch->timeout_event()->timer_set) {
+      ngx_del_timer(fetch->timeout_event());
+      fetch->set_timeout_event(NULL);
+    }
+    fetch->message_handler()->Message(
+        kWarning, "NgxFetch %p: no suitable address for host [%.*s]", fetch,
+        static_cast<int>(resolver_ctx->name.len), resolver_ctx->name.data);
+    fetch->CallbackDone(false);
+  }
+
+  ngx_memzero(&fetch->sin_, sizeof(fetch->sin_));
+
+#if (nginx_version < 1005008)
+  fetch->sin_.sin_addr.s_addr = resolver_ctx->addrs[i];
+#else
+  struct sockaddr_in* sin;
+
+  sin = reinterpret_cast<struct sockaddr_in*>(
+      resolver_ctx->addrs[i].sockaddr);
+
+  fetch->sin_.sin_family = sin->sin_family;
+  fetch->sin_.sin_addr.s_addr = sin->sin_addr.s_addr;
+#endif
+
+  fetch->sin_.sin_family = AF_INET;
+  fetch->sin_.sin_port = htons(fetch->url_.port);
+
+  // Maybe we have Proxy
+  if (0 != fetcher->proxy_.url.len) {
+    fetch->sin_.sin_port = htons(fetcher->proxy_.port);
+  }
+
+  char* ip_address = inet_ntoa(fetch->sin_.sin_addr);
+
+  ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
+                "NgxFetch %p: Resolved host [%V] to [%s]", fetch,
+                &resolver_ctx->name, ip_address);
+
+  fetch->release_resolver();
+
+  if (fetch->InitRequest() != NGX_OK) {
+    fetch->message_handler()->Message(kError, "NgxFetch: InitRequest failed");
+    fetch->CallbackDone(false);
+  }
+}
+
+// Prepare the request data for this fetch, and hook the write event.
+int NgxFetch::InitRequest() {
+  in_ = ngx_create_temp_buf(pool_, 4096);
+  if (in_ == NULL) {
+    return NGX_ERROR;
+  }
+
+  FixUserAgent();
+
+  RequestHeaders* request_headers = async_fetch_->request_headers();
+  ConstStringStarVector v;
+  size_t size = 0;
+  bool have_host = false;
+  GoogleString port;
+
+  response_handler = NgxFetch::HandleStatusLine;
+  int rc = Connect();
+  if (rc == NGX_AGAIN || rc == NGX_OK) {
+    if (connection_->keepalive()) {
+      request_headers->Add(HttpAttributes::kConnection,
+                           NgxConnection::ka_header);
+    }
+    const char* method = request_headers->method_string();
+    size_t method_len = strlen(method);
+
+    size = (method_len +
+            1 /* for the space */ +
+            url_.uri.len +
+            sizeof(" HTTP/1.0\r\n") - 1);
+
+    for (int i = 0; i < request_headers->NumAttributes(); i++) {
+      // if no explicit host header is given in the request headers,
+      // we need to derive it from the url.
+      if (StringCaseEqual(request_headers->Name(i), "Host")) {
+        have_host = true;
+      }
+
+      // name: value\r\n
+      size += request_headers->Name(i).length()
+          + request_headers->Value(i).length() + 4;  // 4 for ": \r\n"
+    }
+
+    if (!have_host) {
+      port = StrCat(":", IntegerToString(url_.port));
+      // for "Host: " + host + ":" + port + "\r\n"
+      size += url_.host.len + 8 + port.size();
+    }
+
+    size += 2;  // "\r\n";
+    out_ = ngx_create_temp_buf(pool_, size);
+
+    if (out_ == NULL) {
+      return NGX_ERROR;
+    }
+
+    out_->last = ngx_cpymem(out_->last, method, method_len);
+    out_->last = ngx_cpymem(out_->last, " ", 1);
+    out_->last = ngx_cpymem(out_->last, url_.uri.data, url_.uri.len);
+    out_->last = ngx_cpymem(out_->last, " HTTP/1.0\r\n", 11);
+
+    if (!have_host) {
+      out_->last = ngx_cpymem(out_->last, "Host: ", 6);
+      out_->last = ngx_cpymem(out_->last, url_.host.data, url_.host.len);
+      out_->last = ngx_cpymem(out_->last, port.c_str(), port.size());
+      out_->last = ngx_cpymem(out_->last, "\r\n", 2);
+    }
+
+    for (int i = 0; i < request_headers->NumAttributes(); i++) {
+      const GoogleString& name = request_headers->Name(i);
+      const GoogleString& value = request_headers->Value(i);
+      out_->last = ngx_cpymem(out_->last, name.c_str(), name.length());
+      *(out_->last++) = ':';
+      *(out_->last++) = ' ';
+      out_->last = ngx_cpymem(out_->last, value.c_str(), value.length());
+      *(out_->last++) = CR;
+      *(out_->last++) = LF;
+    }
+    *(out_->last++) = CR;
+    *(out_->last++) = LF;
+    if (rc == NGX_AGAIN) {
+      return NGX_OK;
+    }
+  } else if (rc < NGX_OK) {
+    return rc;
+  }
+  CHECK(rc == NGX_OK);
+  NgxFetch::ConnectionWriteHandler(connection_->c_->write);
+  return NGX_OK;
+}
+
+int NgxFetch::Connect() {
+  ngx_peer_connection_t pc;
+  ngx_memzero(&pc, sizeof(pc));
+  pc.sockaddr = (struct sockaddr*)&sin_;
+  pc.socklen = sizeof(struct sockaddr_in);
+  pc.name = &url_.host;
+
+  // get callback is dummy function, it just returns NGX_OK
+  pc.get = ngx_event_get_peer;
+  pc.log_error = NGX_ERROR_ERR;
+  pc.log = fetcher_->log_;
+  pc.rcvbuf = -1;
+
+
+  connection_ = NgxConnection::Connect(&pc, message_handler(),
+                                       fetcher_->max_keepalive_requests_);
+  ngx_log_error(NGX_LOG_DEBUG, fetcher_->log_, 0,
+                "NgxFetch %p Connect() connection %p for [%s]",
+                this, connection_, str_url());
+
+  if (connection_ == NULL) {
+    return NGX_ERROR;
+  }
+
+  connection_->c_->write->handler = NgxFetch::ConnectionWriteHandler;
+  connection_->c_->read->handler = NgxFetch::ConnectionReadHandler;
+  connection_->c_->data = this;
+
+  // Timer set in Init() is still in effect.
+  return NGX_OK;
+}
+
+// When the fetch sends the request completely, it will hook the read event,
+// and prepare to parse the response. Timer set in Init() is still in effect.
+void NgxFetch::ConnectionWriteHandler(ngx_event_t* wev) {
+  ngx_connection_t* c = static_cast<ngx_connection_t*>(wev->data);
+  NgxFetch* fetch = static_cast<NgxFetch*>(c->data);
+  ngx_buf_t* out = fetch->out_;
+  bool ok = true;
+  while (wev->ready && out->pos < out->last) {
+    int n = c->send(c, out->pos, out->last - out->pos);
+    ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
+                  "NgxFetch %p: ConnectionWriteHandler "
+                  "send result %d", fetch, n);
+
+    if (n >= 0) {
+      out->pos += n;
+    } else if (n == NGX_AGAIN) {
+      break;
+    } else {
+      ok = false;
+      break;
+    }
+  }
+
+  if (ok) {
+    if (out->pos == out->last) {
+      ok = ngx_handle_read_event(c->read, 0) == NGX_OK;
+    } else {
+      ok = ngx_handle_write_event(c->write, 0) == NGX_OK;
+    }
+  }
+
+  if (!ok) {
+    fetch->message_handler()->Message(
+        kWarning, "NgxFetch %p: failed to hook next event", fetch);
+    c->error = 1;
+    fetch->CallbackDone(false);
+  }
+}
+
+// Timer set in Init() is still in effect.
+void NgxFetch::ConnectionReadHandler(ngx_event_t* rev) {
+  ngx_connection_t* c = static_cast<ngx_connection_t*>(rev->data);
+  NgxFetch* fetch = static_cast<NgxFetch*>(c->data);
+  bool ok = true;
+
+  while(rev->ready) {
+    int n = c->recv(
+        c, fetch->in_->start, fetch->in_->end - fetch->in_->start);
+
+    ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
+                  "NgxFetch %p: ConnectionReadHandler "
+                  "recv result %d", fetch, n);
+
+    if (n == NGX_AGAIN) {
+      break;
+    } else if (n == 0) {
+      // If the content length was not known, we assume that we have read
+      // all if we at least parsed the headers.
+      // If we do know the content length, having a mismatch on the bytes read
+      // will be interpreted as an error.
+      ok = (fetch->content_length_known_ && fetch->content_length_ == fetch->bytes_received_)
+          || fetch->parser_.headers_complete();
+      fetch->done_ = true;
+      break;
+    } else if (n > 0) {
+      fetch->in_->pos = fetch->in_->start;
+      fetch->in_->last = fetch->in_->start + n;
+      ok = fetch->response_handler(c);
+      if (fetch->done_ || !ok) {
+        break;
+      }
+    }
+  }
+
+  if (!ok) {
+    fetch->CallbackDone(false);
+  } else if (fetch->done_) {
+    fetch->CallbackDone(true);
+  } else if (ngx_handle_read_event(rev, 0) != NGX_OK) {
+    fetch->CallbackDone(false);
+  }
+}
+
+// Parse the status line: "HTTP/1.1 200 OK\r\n"
+bool NgxFetch::HandleStatusLine(ngx_connection_t* c) {
+  NgxFetch* fetch = static_cast<NgxFetch*>(c->data);
+  ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
+                "NgxFetch %p: Handle status line", fetch);
+
+  // This function only works after Nginx-1.1.4. Before nginx-1.1.4,
+  // ngx_http_parse_status_line didn't save http_version.
+  ngx_int_t n = ngx_http_parse_status_line(fetch->r_, fetch->in_,
+                                           fetch->status_);
+  if (n == NGX_ERROR) {  // parse status line error
+    fetch->message_handler()->Message(
+        kWarning, "NgxFetch: failed to parse status line");
+    return false;
+  } else if (n == NGX_AGAIN) {  // not completed
+    return true;
+  }
+
+  ResponseHeaders* response_headers =
+      fetch->async_fetch_->response_headers();
+  response_headers->SetStatusAndReason(
+      static_cast<HttpStatus::Code>(fetch->get_status_code()));
+  response_headers->set_major_version(fetch->get_major_version());
+  response_headers->set_minor_version(fetch->get_minor_version());
+
+  fetch->in_->pos += n;
+  fetch->set_response_handler(NgxFetch::HandleHeader);
+  if ((fetch->in_->last - fetch->in_->pos) > 0) {
+    return fetch->response_handler(c);
+  }
+  return true;
+}
+
+// Parse the HTTP headers
+bool NgxFetch::HandleHeader(ngx_connection_t* c) {
+  NgxFetch* fetch = static_cast<NgxFetch*>(c->data);
+  char* data = reinterpret_cast<char*>(fetch->in_->pos);
+  size_t size = fetch->in_->last - fetch->in_->pos;
+  size_t n = fetch->parser_.ParseChunk(StringPiece(data, size),
+                                       fetch->message_handler_);
+
+  ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
+                "NgxFetch %p: Handle headers", fetch);
+
+  if (n > size) {
+    return false;
+  } else if (fetch->parser_.headers_complete()) {
+    // TODO(oschaaf): We should also check if the request method was HEAD
+    // - but I don't think PSOL uses that at this point.
+    if (fetch->get_status_code() == 304 || fetch->get_status_code() == 204) {
+      fetch->done_ = true;
+    } else if (fetch->async_fetch_->response_headers()->FindContentLength(
+            &fetch->content_length_)) {
+      if (fetch->content_length_ < 0) {
+        fetch->message_handler_->Message(
+            kError, "Negative content-length in response header");
+        return false;
+      } else {
+        fetch->content_length_known_ = true;
+        if (fetch->content_length_ == 0) {
+          fetch->done_ = true;
+        }
+      }
+    }
+
+    if (fetch->fetcher_->track_original_content_length()
+        && fetch->content_length_known_) {
+      fetch->async_fetch_->response_headers()->SetOriginalContentLength(
+          fetch->content_length_);
+    }
+
+    fetch->in_->pos += n;
+    if (!fetch->done_) {
+      fetch->set_response_handler(NgxFetch::HandleBody);
+      if ((fetch->in_->last - fetch->in_->pos) > 0) {
+        return fetch->response_handler(c);
+      }
+    }
+  } else {
+    fetch->in_->pos += n;
+  }
+  return true;
+}
+
+// Read the response body
+bool NgxFetch::HandleBody(ngx_connection_t* c) {
+  NgxFetch* fetch = static_cast<NgxFetch*>(c->data);
+  char* data = reinterpret_cast<char*>(fetch->in_->pos);
+  size_t size = fetch->in_->last - fetch->in_->pos;
+
+  fetch->bytes_received_add(size);
+
+  ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
+                "NgxFetch %p: Handle body (%d bytes)", fetch, size);
+
+  if ( fetch->async_fetch_->Write(StringPiece(data, size),
+                                  fetch->message_handler()) ) {
+    if (fetch->bytes_received_ == fetch->content_length_) {
+      fetch->done_ = true;
+    }
+    fetch->in_->pos += size;
+  } else {
+    ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
+                  "NgxFetch %p: async fetch write failure", fetch);
+    return false;
+  }
+  return true;
+}
+
+void NgxFetch::TimeoutHandler(ngx_event_t* tev) {
+  NgxFetch* fetch = static_cast<NgxFetch*>(tev->data);
+  ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
+                "NgxFetch %p: TimeoutHandler called", fetch);
+  fetch->CallbackDone(false);
+}
+
+void NgxFetch::FixUserAgent() {
+  GoogleString user_agent;
+  ConstStringStarVector v;
+  RequestHeaders* request_headers = async_fetch_->request_headers();
+  if (request_headers->Lookup(HttpAttributes::kUserAgent, &v)) {
+    for (size_t i = 0, n = v.size(); i < n; i++) {
+      if (i != 0) {
+        user_agent += " ";
+      }
+
+      if (v[i] != NULL) {
+        user_agent += *(v[i]);
+      }
+    }
+    request_headers->RemoveAll(HttpAttributes::kUserAgent);
+  }
+  if (user_agent.empty()) {
+    user_agent += "NgxNativeFetcher";
+  }
+  GoogleString version = StrCat(
+      " (", kModPagespeedSubrequestUserAgent,
+      "/" MOD_PAGESPEED_VERSION_STRING "-" LASTCHANGE_STRING ")");
+  if (!StringPiece(user_agent).ends_with(version)) {
+    user_agent += version;
+  }
+  request_headers->Add(HttpAttributes::kUserAgent, user_agent);
+}
+
+}  // namespace net_instaweb

ngx_pagespeed-latest-beta/src/ngx_fetch.h

@@ -0,0 +1,210 @@
+/*
+ * Copyright 2012 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: x.dinic@gmail.com (Junmin Xiong)
+//
+// PageSpeed needs some way to talk to the internet and request resources.  For
+// example, if it's optimizing www.example.com/index.html and it sees html with
+// <img src="//images.example.com/cat.jpg"> and images.example.com is authorized
+// for rewriting in the config, then it needs to fetch cat.jpg from
+// images.example.com and optimize it.  In apache (always) and nginx (by
+// default) we use a fetcher called "serf".  This works fine, but it does run
+// its own event loop.  To be more efficient, this is a "native" fetcher that
+// uses nginx's event loop.
+//
+// The fetch is started by the main thread. It will fetch the remote resource
+// from the specific url asynchronously.
+
+#ifndef NET_INSTAWEB_NGX_FETCH_H_
+#define NET_INSTAWEB_NGX_FETCH_H_
+
+extern "C" {
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_http.h>
+}
+
+#include "ngx_url_async_fetcher.h"
+#include <vector>
+#include "net/instaweb/http/public/url_async_fetcher.h"
+#include "pagespeed/kernel/base/basictypes.h"
+#include "pagespeed/kernel/base/pool.h"
+#include "pagespeed/kernel/base/string.h"
+#include "pagespeed/kernel/http/response_headers.h"
+#include "pagespeed/kernel/http/response_headers_parser.h"
+#include "pagespeed/kernel/thread/pthread_mutex.h"
+
+
+namespace net_instaweb {
+
+typedef bool (*response_handler_pt)(ngx_connection_t* c);
+
+class NgxUrlAsyncFetcher;
+class NgxConnection;
+
+class NgxConnection : public PoolElement<NgxConnection> {
+ public:
+  NgxConnection(MessageHandler* handler, int max_keepalive_requests);
+  ~NgxConnection();
+  void SetSock(u_char *sockaddr, socklen_t socklen) {
+    socklen_ = socklen;
+    ngx_memcpy(&sockaddr_, sockaddr, socklen);
+  }
+  // Close ensures that NgxConnection deletes itself at the appropriate time,
+  // which can be after receiving a non-keepalive response, or when the remote
+  // server closes the connection when the NgxConnection is pooled and idle.
+  void Close();
+
+  // Once keepalive is disabled, it can't be toggled back on.
+  void set_keepalive(bool k) { keepalive_ = keepalive_ && k; }
+  bool keepalive() { return keepalive_; }
+
+  typedef Pool<NgxConnection> NgxConnectionPool;
+
+  static NgxConnection* Connect(ngx_peer_connection_t* pc,
+                                MessageHandler* handler,
+                                int max_keepalive_requests);
+  static void IdleWriteHandler(ngx_event_t* ev);
+  static void IdleReadHandler(ngx_event_t* ev);
+  // Terminate will cleanup any idle connections upon shutdown.
+  static void Terminate();
+
+  static NgxConnectionPool connection_pool;
+  static PthreadMutex connection_pool_mutex;
+
+  // c_ is owned by NgxConnection and freed in ::Close()
+  ngx_connection_t* c_;
+  static const int64 keepalive_timeout_ms;
+  static const GoogleString ka_header;
+
+ private:
+  int max_keepalive_requests_;
+  bool keepalive_;
+  socklen_t socklen_;
+  u_char sockaddr_[NGX_SOCKADDRLEN];
+  MessageHandler* handler_;
+
+  DISALLOW_COPY_AND_ASSIGN(NgxConnection);
+};
+
+class NgxFetch : public PoolElement<NgxFetch> {
+ public:
+  NgxFetch(const GoogleString& url,
+           AsyncFetch* async_fetch,
+           MessageHandler* message_handler,
+           ngx_log_t* log);
+  ~NgxFetch();
+
+  // Start the fetch.
+  bool Start(NgxUrlAsyncFetcher* fetcher);
+  // Show the completed url, for logging purposes.
+  const char* str_url();
+  // This fetch task is done. Call Done() on the async_fetch. It will copy the
+  // buffer to cache.
+  void CallbackDone(bool success);
+
+  // Show the bytes received.
+  size_t bytes_received();
+  void bytes_received_add(int64 x);
+  int64 fetch_start_ms();
+  void set_fetch_start_ms(int64 start_ms);
+  int64 fetch_end_ms();
+  void set_fetch_end_ms(int64 end_ms);
+  MessageHandler* message_handler();
+
+  int get_major_version() {
+    return static_cast<int>(status_->http_version / 1000);
+  }
+  int get_minor_version() {
+    return static_cast<int>(status_->http_version % 1000);
+  }
+  int get_status_code() {
+    return static_cast<int>(status_->code);
+  }
+  ngx_event_t* timeout_event() {
+    return timeout_event_;
+  }
+  void set_timeout_event(ngx_event_t* x) {
+    timeout_event_ = x;
+  }
+  void release_resolver() {
+    if (resolver_ctx_ != NULL && resolver_ctx_ != NGX_NO_RESOLVER) {
+      ngx_resolve_name_done(resolver_ctx_);
+      resolver_ctx_ = NULL;
+    }
+  }
+
+ private:
+  response_handler_pt response_handler;
+  // Do the initialized work and start the resolver work.
+  bool Init();
+  bool ParseUrl();
+  // Prepare the request and write it to remote server.
+  int InitRequest();
+  // Create the connection with remote server.
+  int Connect();
+  void set_response_handler(response_handler_pt handler) {
+    response_handler = handler;
+  }
+  // Only the Static functions could be used in callbacks.
+  static void ResolveDoneHandler(ngx_resolver_ctx_t* ctx);
+  // Write the request.
+  static void ConnectionWriteHandler(ngx_event_t* wev);
+  // Wait for the response.
+  static void ConnectionReadHandler(ngx_event_t* rev);
+  // Read and parse the first status line.
+  static bool HandleStatusLine(ngx_connection_t* c);
+  // Read and parse the HTTP headers.
+  static bool HandleHeader(ngx_connection_t* c);
+  // Read the response body.
+  static bool HandleBody(ngx_connection_t* c);
+  // Cancel the fetch when it's timeout.
+  static void TimeoutHandler(ngx_event_t* tev);
+
+  // Add the pagespeed User-Agent.
+  void FixUserAgent();
+  void FixHost();
+
+  const GoogleString str_url_;
+  ngx_url_t url_;
+  NgxUrlAsyncFetcher* fetcher_;
+  AsyncFetch* async_fetch_;
+  ResponseHeadersParser parser_;
+  MessageHandler* message_handler_;
+  int64 bytes_received_;
+  int64 fetch_start_ms_;
+  int64 fetch_end_ms_;
+  bool done_;
+  int64 content_length_;
+  bool content_length_known_;
+
+  struct sockaddr_in sin_;
+  ngx_log_t* log_;
+  ngx_buf_t* out_;
+  ngx_buf_t* in_;
+  ngx_pool_t* pool_;
+  ngx_http_request_t* r_;
+  ngx_http_status_t* status_;
+  ngx_event_t* timeout_event_;
+  NgxConnection* connection_;
+  ngx_resolver_ctx_t* resolver_ctx_;
+
+  DISALLOW_COPY_AND_ASSIGN(NgxFetch);
+};
+
+}  // namespace net_instaweb
+
+#endif  // NET_INSTAWEB_NGX_FETCH_H_

ngx_pagespeed-latest-beta/src/ngx_gzip_setter.cc

@@ -0,0 +1,403 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: kspoelstra@we-amp.com (Kees Spoelstra)
+
+#include "ngx_gzip_setter.h"
+
+#include <ngx_conf_file.h>
+
+namespace net_instaweb {
+
+NgxGZipSetter g_gzip_setter;
+
+extern "C" {
+  // These functions replace the setters for:
+  //   gzip
+  //   gzip_types
+  //   gzip_http_version
+  //   gzip_vary
+  //
+  // If these functions are called it means there is an explicit gzip
+  // configuration. The gzip configuration set by pagespeed is then rolled
+  // back and pagespeed will stop enabling gzip automatically.
+  char* ngx_gzip_redirect_conf_set_flag_slot(
+      ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
+    if (g_gzip_setter.enabled()) {
+      g_gzip_setter.RollBackAndDisable(cf);
+    }
+    char* ret = ngx_conf_set_flag_slot(cf, cmd, conf);
+    return ret;
+  }
+
+  char* ngx_gzip_redirect_http_types_slot(
+      ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
+    if (g_gzip_setter.enabled()) {
+      g_gzip_setter.RollBackAndDisable(cf);
+    }
+    char* ret = ngx_http_types_slot(cf, cmd, conf);
+    return ret;
+  }
+
+  char* ngx_gzip_redirect_conf_set_enum_slot(
+      ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
+    if (g_gzip_setter.enabled()) {
+      g_gzip_setter.RollBackAndDisable(cf);
+    }
+    char* ret = ngx_conf_set_enum_slot(cf, cmd, conf);
+    return ret;
+  }
+  char* ngx_gzip_redirect_conf_set_bitmask_slot(
+      ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
+    if (g_gzip_setter.enabled()) {
+      g_gzip_setter.RollBackAndDisable(cf);
+    }
+    char* ret = ngx_conf_set_bitmask_slot(cf, cmd, conf);
+    return ret;
+  }
+}
+
+NgxGZipSetter::NgxGZipSetter() : enabled_(0) { }
+NgxGZipSetter::~NgxGZipSetter() { }
+
+// Helper functions to determine signature.
+bool HasLocalConfig(ngx_command_t* command) {
+  return (!(command->type & (NGX_DIRECT_CONF|NGX_MAIN_CONF)) &&
+          command->conf == NGX_HTTP_LOC_CONF_OFFSET);
+}
+bool IsNgxFlagCommand(ngx_command_t* command) {
+  return (command->set == ngx_conf_set_flag_slot &&
+          HasLocalConfig(command));
+}
+bool IsNgxHttpTypesCommand(ngx_command_t* command) {
+  return (command->set == ngx_http_types_slot &&
+          HasLocalConfig(command));
+}
+bool IsNgxEnumCommand(ngx_command_t* command) {
+  return (command->set == ngx_conf_set_enum_slot &&
+          HasLocalConfig(command));
+}
+bool IsNgxBitmaskCommand(ngx_command_t* command) {
+  return (command->set == ngx_conf_set_bitmask_slot &&
+          HasLocalConfig(command));
+}
+
+// Initialize the NgxGzipSetter.
+// Find the gzip, gzip_vary, gzip_http_version and gzip_types commands in the
+// gzip module. Enable if the signature of the zip command matches with what we
+// trust. Also sets up redirects for the configurations. These redirect handle
+// a rollback if expicit configuration is found.
+// If commands are not found the method will inform the user by logging.
+void NgxGZipSetter::Init(ngx_conf_t* cf) {
+#if (NGX_HTTP_GZIP)
+  bool gzip_signature_mismatch = false;
+  bool other_signature_mismatch = false;
+  for (int m = 0; ngx_modules[m] != NULL; m++) {
+    if (ngx_modules[m]->commands != NULL) {
+      for (int c = 0; ngx_modules[m]->commands[c].name.len; c++) {
+        ngx_command_t* current_command =& ngx_modules[m]->commands[c];
+
+        // We look for the gzip command, and the exact signature we trust
+        // this means configured as an config location offset
+        // and a ngx_flag_t setter.
+        // Also see:
+        //   ngx_conf_handler in ngx_conf_file.c
+        //   ngx_http_gzip_filter_commands in ngx_http_gzip_filter.c
+        if (gzip_command_.command_ == NULL &&
+            STR_EQ_LITERAL(current_command->name, "gzip")) {
+          if (IsNgxFlagCommand(current_command)) {
+            current_command->set = ngx_gzip_redirect_conf_set_flag_slot;
+            gzip_command_.command_ = current_command;
+            gzip_command_.module_ = ngx_modules[m];
+            enabled_ = 1;
+          } else {
+            ngx_conf_log_error(
+                NGX_LOG_WARN, cf, 0,
+                "pagespeed: cannot set gzip, signature mismatch");
+            gzip_signature_mismatch = true;
+          }
+        }
+
+        if (!gzip_http_version_command_.command_ &&
+            STR_EQ_LITERAL(current_command->name, "gzip_http_version")) {
+          if (IsNgxEnumCommand(current_command)) {
+            current_command->set = ngx_gzip_redirect_conf_set_enum_slot;
+            gzip_http_version_command_.command_ = current_command;
+            gzip_http_version_command_.module_ = ngx_modules[m];
+          } else {
+            ngx_conf_log_error(
+                NGX_LOG_WARN, cf, 0,
+                "pagespeed: cannot set gzip_http_version, signature mismatch");
+            other_signature_mismatch = true;
+          }
+        }
+
+        if (!gzip_proxied_command_.command_ &&
+            STR_EQ_LITERAL(current_command->name, "gzip_proxied")) {
+          if (IsNgxBitmaskCommand(current_command)) {
+            current_command->set = ngx_gzip_redirect_conf_set_bitmask_slot;
+            gzip_proxied_command_.command_ = current_command;
+            gzip_proxied_command_.module_ = ngx_modules[m];
+          } else {
+            ngx_conf_log_error(
+                NGX_LOG_WARN, cf, 0,
+                "pagespeed: cannot set gzip_proxied, signature mismatch");
+            other_signature_mismatch = true;
+          }
+        }
+
+        if (!gzip_http_types_command_.command_ &&
+            STR_EQ_LITERAL(current_command->name, "gzip_types")) {
+          if (IsNgxHttpTypesCommand(current_command)) {
+            current_command->set = ngx_gzip_redirect_http_types_slot;
+            gzip_http_types_command_.command_ = current_command;
+            gzip_http_types_command_.module_ = ngx_modules[m];
+          } else {
+            ngx_conf_log_error(
+                NGX_LOG_WARN, cf, 0,
+                "pagespeed: cannot set gzip_types, signature mismatch");
+            other_signature_mismatch = true;
+          }
+        }
+
+        if (!gzip_vary_command_.command_ &&
+            STR_EQ_LITERAL(current_command->name, "gzip_vary")) {
+          if (IsNgxFlagCommand(current_command)) {
+            current_command->set = ngx_gzip_redirect_conf_set_flag_slot;
+            gzip_vary_command_.command_ = current_command;
+            gzip_vary_command_.module_ = ngx_modules[m];
+          } else {
+            ngx_conf_log_error(
+                NGX_LOG_WARN, cf, 0,
+                "pagespeed: cannot set gzip_vary, signature mismatch");
+            other_signature_mismatch = true;
+          }
+        }
+      }
+    }
+  }
+  if (gzip_signature_mismatch) {
+    return;  // Already logged error.
+  } else if (!enabled_) {
+    // Looked through all the available commands and didn't find the "gzip" one.
+    ngx_conf_log_error(
+        NGX_LOG_WARN, cf, 0, "pagespeed: cannot set gzip, command not found");
+    return;
+  } else if (other_signature_mismatch) {
+    return;  // Already logged error.
+  } else if (!gzip_vary_command_.command_) {
+    ngx_conf_log_error(
+        NGX_LOG_WARN, cf, 0, "pagespeed: missing gzip_vary");
+    return;
+  } else if (!gzip_http_types_command_.command_) {
+    ngx_conf_log_error(
+        NGX_LOG_WARN, cf, 0, "pagespeed: missing gzip_types");
+    return;
+  } else if (!gzip_http_version_command_.command_) {
+    ngx_conf_log_error(
+        NGX_LOG_WARN, cf, 0, "pagespeed: missing gzip_http_version");
+    return;
+  } else if (!gzip_proxied_command_.command_) {
+    ngx_conf_log_error(
+        NGX_LOG_WARN, cf, 0, "pagespeed: missing gzip_proxied");
+    return;
+  } else {
+    return;  // Success.
+  }
+#else
+  ngx_conf_log_error(
+      NGX_LOG_WARN, cf, 0, "pagespeed: gzip not compiled into nginx");
+  return;
+#endif
+}
+
+void* ngx_command_ctx::GetConfPtr(ngx_conf_t* cf) {
+  return GetModuleConfPtr(cf) + command_->offset;
+}
+
+char* ngx_command_ctx::GetModuleConfPtr(ngx_conf_t* cf) {
+  return reinterpret_cast<char*>(
+      ngx_http_conf_get_module_loc_conf(cf, (*(module_))));
+}
+
+void NgxGZipSetter::SetNgxConfFlag(ngx_conf_t* cf,
+                                   ngx_command_ctx* command_ctx,
+                                   ngx_flag_t value) {
+  ngx_flag_t* flag = reinterpret_cast<ngx_flag_t*>(command_ctx->GetConfPtr(cf));
+  *flag = value;
+  // Save the flag position for possible rollback.
+  ngx_flags_set_.push_back(flag);
+}
+
+void NgxGZipSetter::SetNgxConfEnum(ngx_conf_t* cf,
+                                   ngx_command_ctx* command_ctx,
+                                   ngx_uint_t value) {
+  ngx_uint_t* enum_to_set =
+      reinterpret_cast<ngx_uint_t*>(command_ctx->GetConfPtr(cf));
+  *enum_to_set = value;
+  ngx_uint_set_.push_back(enum_to_set);
+}
+
+void NgxGZipSetter::SetNgxConfBitmask(ngx_conf_t* cf,
+                                      ngx_command_ctx* command_ctx,
+                                      ngx_uint_t value) {
+  ngx_uint_t* enum_to_set =
+      reinterpret_cast<ngx_uint_t*>(command_ctx->GetConfPtr(cf));
+  *enum_to_set = value;
+  ngx_uint_set_.push_back(enum_to_set);
+}
+
+// These are the content types we want to compress.
+ngx_str_t gzip_http_types[] = {
+  ngx_string("application/ecmascript"),
+  ngx_string("application/javascript"),
+  ngx_string("application/json"),
+  ngx_string("application/pdf"),
+  ngx_string("application/postscript"),
+  ngx_string("application/x-javascript"),
+  ngx_string("image/svg+xml"),
+  ngx_string("text/css"),
+  ngx_string("text/csv"),
+  // ngx_string("text/html"),  // This is the default implied value.
+  ngx_string("text/javascript"),
+  ngx_string("text/plain"),
+  ngx_string("text/xml"),
+  ngx_null_string  // Indicates end of array.
+};
+
+gzs_enable_result NgxGZipSetter::SetGZipForLocation(ngx_conf_t* cf,
+                                                    bool value) {
+  if (!enabled_) {
+    return kEnableGZipNotEnabled;
+  }
+  if (gzip_command_.command_) {
+    SetNgxConfFlag(cf, &gzip_command_, value);
+  }
+  return kEnableGZipOk;
+}
+
+void NgxGZipSetter::EnableGZipForLocation(ngx_conf_t* cf) {
+  if (!enabled_) {
+    return;
+  }
+
+  // When we get called twice for the same location{}, we ignore the second call
+  // to prevent adding duplicate gzip http types and so on.
+  ngx_flag_t* flag =
+      reinterpret_cast<ngx_flag_t*>(gzip_command_.GetConfPtr(cf));
+  if (*flag == 1) {
+    return;
+  }
+  SetGZipForLocation(cf, true);
+  if (gzip_vary_command_.command_) {
+    SetNgxConfFlag(cf, &gzip_vary_command_, 1);
+  }
+  if (gzip_http_version_command_.command_) {
+    SetNgxConfEnum(cf, &gzip_http_version_command_, NGX_HTTP_VERSION_10);
+  }
+  if (gzip_proxied_command_.command_) {
+    SetNgxConfBitmask(
+        cf, &gzip_http_version_command_, NGX_HTTP_GZIP_PROXIED_ANY);
+  }
+
+  // This is actually the most prone to future API changes, because gzip_types
+  // is not a simple type like ngx_flag_t. The signature check should be enough
+  // to prevent problems.
+  AddGZipHTTPTypes(cf);
+  return;
+}
+
+void NgxGZipSetter::AddGZipHTTPTypes(ngx_conf_t* cf) {
+  if (gzip_http_types_command_.command_) {
+    // Following should not happen, but if it does return gracefully.
+    if (cf->args->nalloc < 2) {
+      ngx_conf_log_error(NGX_LOG_WARN, cf, 0,
+                         "pagespeed: unexpected small cf->args in gzip_types");
+      return;
+    }
+
+    ngx_command_t* command = gzip_http_types_command_.command_;
+    char* gzip_conf = reinterpret_cast<char* >(
+        gzip_http_types_command_.GetModuleConfPtr(cf));
+
+    // Backup the old settings.
+    ngx_str_t old_elt0 = reinterpret_cast<ngx_str_t*>(cf->args->elts)[0];
+    ngx_str_t old_elt1 = reinterpret_cast<ngx_str_t*>(cf->args->elts)[1];
+    ngx_uint_t old_nelts = cf->args->nelts;
+
+    // Setup first arg.
+    ngx_str_t gzip_types_string = ngx_string("gzip_types");
+    reinterpret_cast<ngx_str_t*>(cf->args->elts)[0] = gzip_types_string;
+    cf->args->nelts = 2;
+
+    ngx_str_t* http_types = gzip_http_types;
+    while (http_types->data) {
+      ngx_str_t d;
+      // We allocate the http type on the configuration pool and actually
+      // leak this if we rollback. This does not seem to be a big problem,
+      // because nginx also allocates tokens in ngx_conf_file.c and does not
+      // free them. This way they can be used safely by configurations.
+      // We must use a copy of gzip_http_types array here because nginx will
+      // manipulate the values.
+      // TODO(kspoelstra): better would be to allocate once on init and not
+      // every time we enable gzip. This needs further investigation, sharing
+      // tokens might be problematic.
+      // For now I think it is not a large problem. This might add up in case
+      // of a large multi server/location config with a lot of "pagespeed on"
+      // directives.
+      // Estimates are 300-400KB for 1000 times "pagespeed on".
+      d.data = reinterpret_cast<u_char*>(
+          ngx_pnalloc(cf->pool, http_types->len + 1));
+      snprintf(reinterpret_cast<char*>(d.data), http_types->len + 1, "%s",
+               reinterpret_cast<const char*>(http_types->data));
+      d.len = http_types->len;
+      reinterpret_cast<ngx_str_t*>(cf->args->elts)[1] = d;
+      // Call the original setter.
+      ngx_http_types_slot(cf, command, gzip_conf);
+      http_types++;
+    }
+
+    // Restore args.
+    cf->args->nelts = old_nelts;
+    reinterpret_cast<ngx_str_t*>(cf->args->elts)[1] = old_elt1;
+    reinterpret_cast<ngx_str_t*>(cf->args->elts)[0] = old_elt0;
+
+    // Backup configuration location for rollback.
+    ngx_httptypes_set_.push_back(gzip_conf + command->offset);
+  }
+}
+
+void NgxGZipSetter::RollBackAndDisable(ngx_conf_t* cf) {
+  ngx_conf_log_error(NGX_LOG_INFO, cf, 0,
+                     "pagespeed: rollback gzip, explicit configuration");
+  for (std::vector<ngx_flag_t*>::iterator i = ngx_flags_set_.begin();
+       i != ngx_flags_set_.end(); ++i) {
+    *(*i)=NGX_CONF_UNSET;
+  }
+  for (std::vector<ngx_uint_t*>::iterator i = ngx_uint_set_.begin();
+       i != ngx_uint_set_.end(); ++i) {
+    *(*i)=NGX_CONF_UNSET_UINT;
+  }
+  for (std::vector<void*>::iterator i = ngx_httptypes_set_.begin();
+       i != ngx_httptypes_set_.end(); ++i) {
+    ngx_array_t** type_array = reinterpret_cast<ngx_array_t**>(*i);
+    ngx_array_destroy(*type_array);
+    *type_array = NULL;
+  }
+  enabled_ = 0;
+}
+
+}  // namespace net_instaweb

ngx_pagespeed-latest-beta/src/ngx_gzip_setter.h

@@ -0,0 +1,124 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: kspoelstra@we-amp.com (Kees Spoelstra)
+
+/*
+ * NgxGZipSetter sets up gzip for pagespeed
+ * with the following configuration:
+ * gzip  on;
+ * gzip_vary on;
+ * gzip_types application/ecmascript;
+ * gzip_types application/javascript;
+ * gzip_types application/json;
+ * gzip_types application/pdf;
+ * gzip_types application/postscript;
+ * gzip_types application/x-javascript;
+ * gzip_types image/svg+xml;
+ * gzip_types text/css;
+ * gzip_types text/csv;
+ * gzip_types text/javascript;
+ * gzip_types text/plain;
+ * gzip_types text/xml;
+ * gzip_http_version 1.0;
+ *
+ * If there is an explicit gzip configuration in the nginx.conf
+ * pagespeed will rollback the set configuration and let the
+ * user decide what the configuration will be.
+ *
+ * It manipulates the configuration by manipulating ngx_flag_t
+ * and ngx_uint_t settings directly and using the nginx setter for
+ * gzip_http_types.
+ * This is probably a safe way to do it. If this mechanism
+ * changes all non nginx module setup & configuration will
+ * fail.
+ */
+
+#ifndef NGX_GZIP_SETTER_H_
+#define NGX_GZIP_SETTER_H_
+
+extern "C" {
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_http.h>
+}
+
+#include <vector>
+
+#include "ngx_pagespeed.h"
+
+#include "pagespeed/kernel/base/basictypes.h"
+
+namespace net_instaweb {
+
+// We need this class because configuration for gzip is in different modules, so
+// just saving the command will not work.
+class ngx_command_ctx {
+ public:
+  ngx_command_ctx() : command_(NULL), module_(NULL) { }
+  void* GetConfPtr(ngx_conf_t* cf);
+  char* GetModuleConfPtr(ngx_conf_t* cf);
+  ngx_command_t* command_;
+  ngx_module_t* module_;
+};
+
+enum gzs_enable_result {
+  kEnableGZipOk,
+  kEnableGZipPartial,
+  kEnableGZipNotEnabled
+};
+
+class NgxGZipSetter {
+  std::vector<ngx_flag_t*> ngx_flags_set_;
+  std::vector<ngx_uint_t*> ngx_uint_set_;
+  std::vector<void*> ngx_httptypes_set_;
+  ngx_command_ctx gzip_command_;
+  ngx_command_ctx gzip_http_types_command_;
+  ngx_command_ctx gzip_proxied_command_;
+  ngx_command_ctx gzip_vary_command_;
+  ngx_command_ctx gzip_http_version_command_;
+  bool enabled_;
+
+ public:
+  NgxGZipSetter();
+  ~NgxGZipSetter();
+  void Init(ngx_conf_t* cf);
+
+  void SetNgxConfFlag(ngx_conf_t* cf,
+                      ngx_command_ctx* command_ctx,
+                      ngx_flag_t value);
+  void SetNgxConfEnum(ngx_conf_t* cf,
+                      ngx_command_ctx* command_ctx,
+                      ngx_uint_t value);
+  void SetNgxConfBitmask(ngx_conf_t* cf,
+                         ngx_command_ctx* command_ctx,
+                         ngx_uint_t value);
+  void EnableGZipForLocation(ngx_conf_t* cf);
+  gzs_enable_result SetGZipForLocation(ngx_conf_t* cf, bool value);
+  void AddGZipHTTPTypes(ngx_conf_t* cf);
+  void RollBackAndDisable(ngx_conf_t* cf);
+
+  bool enabled() { return enabled_; }
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(NgxGZipSetter);
+};
+
+extern NgxGZipSetter g_gzip_setter;
+
+}  // namespace net_instaweb
+
+#endif  // NGX_GZIP_SETTER_H_

ngx_pagespeed-latest-beta/src/ngx_list_iterator.cc

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2013 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+
+#include "ngx_list_iterator.h"
+
+namespace net_instaweb {
+
+NgxListIterator::NgxListIterator(ngx_list_part_t* part) :
+    part_(part),
+    index_within_part_(0) {}
+
+ngx_table_elt_t* NgxListIterator::Next() {
+  if (index_within_part_ >= part_->nelts) {
+    if (part_->next == NULL) {
+      return NULL;
+    }
+    part_ = part_->next;
+    index_within_part_ = 0;
+  }
+  ngx_table_elt_t* elts = static_cast<ngx_table_elt_t*>(part_->elts);
+  return &elts[index_within_part_++];  // Intentional post-increment.
+}
+
+}  // namespace net_instaweb

ngx_pagespeed-latest-beta/src/ngx_list_iterator.h

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2013 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+//
+// Simplifies iteration over nginx lists.
+//
+//
+
+#ifndef NGX_LIST_ITERATOR_H_
+#define NGX_LIST_ITERATOR_H_
+
+extern "C" {
+#include <ngx_http.h>
+}
+
+#include "base/basictypes.h"
+
+namespace net_instaweb {
+
+class NgxListIterator {
+ public:
+  explicit NgxListIterator(ngx_list_part_t* part);
+
+  // Return the next element of the list if there is one, NULL otherwise.
+  ngx_table_elt_t* Next();
+
+ private:
+  ngx_list_part_t* part_;
+  ngx_uint_t index_within_part_;
+
+  DISALLOW_COPY_AND_ASSIGN(NgxListIterator);
+};
+
+}  // namespace net_instaweb
+
+#endif  // NGX_LIST_ITERATOR_H_

ngx_pagespeed-latest-beta/src/ngx_message_handler.cc

@@ -0,0 +1,116 @@
+// Copyright 2013 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Author: oschaaf@gmail.com (Otto van der Schaaf)
+
+#include "ngx_message_handler.h"
+
+#include <signal.h>
+
+#include "net/instaweb/public/version.h"
+#include "pagespeed/kernel/base/abstract_mutex.h"
+#include "pagespeed/kernel/base/debug.h"
+#include "pagespeed/kernel/base/posix_timer.h"
+#include "pagespeed/kernel/base/string_util.h"
+#include "pagespeed/kernel/base/time_util.h"
+#include "pagespeed/kernel/sharedmem/shared_circular_buffer.h"
+
+namespace {
+
+// This will be prefixed to every logged message.
+const char kModuleName[] = "ngx_pagespeed";
+
+// If set, the crash handler will use this to output a backtrace using
+// ngx_log_error.
+ngx_log_t* global_log = NULL;
+
+}  // namespace
+
+extern "C" {
+  static void signal_handler(int sig) {
+    // Try to output the backtrace to the log file. Since this may end up
+    // crashing/deadlocking/etc. we set an alarm() to abort us if it comes to
+    // that.
+    alarm(2);
+    if (global_log != NULL) {
+      ngx_log_error(NGX_LOG_ALERT, global_log, 0, "Trapped signal [%d]\n%s",
+                    sig, net_instaweb::StackTraceString().c_str());
+    } else {
+      fprintf(stderr, "Trapped signal [%d]\n%s\n",
+              sig, net_instaweb::StackTraceString().c_str());
+    }
+    kill(getpid(), SIGKILL);
+  }
+}  // extern "C"
+
+namespace net_instaweb {
+
+NgxMessageHandler::NgxMessageHandler(Timer* timer, AbstractMutex* mutex)
+    : SystemMessageHandler(timer, mutex),
+      log_(NULL) {
+}
+
+// Installs a signal handler for common crash signals, that tries to print
+// out a backtrace.
+void NgxMessageHandler::InstallCrashHandler(ngx_log_t* log) {
+  global_log = log;
+  signal(SIGTRAP, signal_handler);  // On check failures
+  signal(SIGABRT, signal_handler);
+  signal(SIGFPE, signal_handler);
+  signal(SIGSEGV, signal_handler);
+}
+
+ngx_uint_t NgxMessageHandler::GetNgxLogLevel(MessageType type) {
+  switch (type) {
+    case kInfo:
+      return NGX_LOG_INFO;
+    case kWarning:
+      return NGX_LOG_WARN;
+    case kError:
+      return NGX_LOG_ERR;
+    case kFatal:
+      return NGX_LOG_ALERT;
+  }
+
+  // This should never fall through, but some compilers seem to complain if
+  // we don't include this.
+  return NGX_LOG_ALERT;
+}
+
+void NgxMessageHandler::MessageSImpl(MessageType type,
+                                     const GoogleString& message) {
+  if (log_ != NULL) {
+    ngx_uint_t log_level = GetNgxLogLevel(type);
+    ngx_log_error(log_level, log_, 0/*ngx_err_t*/, "[%s %s] %s",
+                  kModuleName, kModPagespeedVersion, message.c_str());
+  } else {
+    GoogleMessageHandler::MessageSImpl(type, message);
+  }
+  AddMessageToBuffer(type, message);
+}
+
+void NgxMessageHandler::FileMessageSImpl(
+    MessageType type, const char* file, int line, const GoogleString& message) {
+  if (log_ != NULL) {
+    ngx_uint_t log_level = GetNgxLogLevel(type);
+    ngx_log_error(log_level, log_, 0/*ngx_err_t*/, "[%s %s] %s:%d:%s",
+                  kModuleName, kModPagespeedVersion, file, line,
+                  message.c_str());
+  } else {
+    GoogleMessageHandler::FileMessageSImpl(type, file, line, message);
+  }
+  AddMessageToBuffer(type, file, line, message);
+}
+
+}  // namespace net_instaweb

ngx_pagespeed-latest-beta/src/ngx_message_handler.h

@@ -0,0 +1,70 @@
+// Copyright 2013 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Author: oschaaf@gmail.com (Otto van der Schaaf)
+
+#ifndef NGX_MESSAGE_HANDLER_H_
+#define NGX_MESSAGE_HANDLER_H_
+
+extern "C" {
+  #include <ngx_auto_config.h>
+#if (NGX_THREADS)
+  #include <ngx_thread.h>
+#endif
+  #include <ngx_core.h>
+  #include <ngx_log.h>
+}
+
+#include <cstdarg>
+
+#include "pagespeed/kernel/base/basictypes.h"
+#include "pagespeed/kernel/base/message_handler.h"
+#include "pagespeed/kernel/base/string.h"
+#include "pagespeed/kernel/base/string_util.h"
+#include "pagespeed/system/system_message_handler.h"
+
+namespace net_instaweb {
+
+class AbstractMutex;
+class Timer;
+
+// Implementation of a message handler that uses ngx_log_error()
+// logging to emit messages, with a fallback to GoogleMessageHandler
+class NgxMessageHandler : public SystemMessageHandler {
+ public:
+  explicit NgxMessageHandler(Timer* timer, AbstractMutex* mutex);
+
+  // Installs a signal handler for common crash signals that tries to print
+  // out a backtrace.
+  static void InstallCrashHandler(ngx_log_t* log);
+
+  void set_log(ngx_log_t* log) { log_ = log; }
+  ngx_log_t* log() { return log_; }
+
+ protected:
+  virtual void MessageSImpl(MessageType type, const GoogleString& message);
+
+  virtual void FileMessageSImpl(MessageType type, const char* file,
+                                int line, const GoogleString& message);
+
+ private:
+  ngx_uint_t GetNgxLogLevel(MessageType type);
+  ngx_log_t* log_;
+
+  DISALLOW_COPY_AND_ASSIGN(NgxMessageHandler);
+};
+
+}  // namespace net_instaweb
+
+#endif  // NGX_MESSAGE_HANDLER_H_

ngx_pagespeed-latest-beta/src/ngx_pagespeed.h

@@ -0,0 +1,138 @@
+/*
+ * Copyright 2012 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+
+#ifndef NGX_PAGESPEED_H_
+#define NGX_PAGESPEED_H_
+
+// We might be compiled with syslog.h, which #defines LOG_INFO and LOG_WARNING
+// as ints.  But logging.h assumes they're usable as names, within their
+// namespace, so we need to #undef them before including logging.h
+#ifdef LOG_INFO
+#undef LOG_INFO
+#endif
+#ifdef LOG_WARNING
+#undef LOG_WARNING
+#endif
+
+extern "C" {
+  #include <ngx_http.h>
+}
+
+#include "base/logging.h"
+#include "pagespeed/kernel/base/string_util.h"
+#include "pagespeed/kernel/http/response_headers.h"
+
+namespace net_instaweb {
+
+class GzipInflater;
+class NgxBaseFetch;
+class ProxyFetch;
+class RewriteDriver;
+class RequestHeaders;
+class ResponseHeaders;
+class InPlaceResourceRecorder;
+
+// Allocate chain links and buffers from the supplied pool, and copy over the
+// data from the string piece.  If the string piece is empty, return
+// NGX_DECLINED immediately unless send_last_buf.
+ngx_int_t string_piece_to_buffer_chain(
+    ngx_pool_t* pool, StringPiece sp,
+    ngx_chain_t** link_ptr, bool send_last_buf, bool send_flush);
+
+StringPiece str_to_string_piece(ngx_str_t s);
+
+// s1: ngx_str_t, s2: string literal
+// true if they're equal, false otherwise
+#define STR_EQ_LITERAL(s1, s2)          \
+    ((s1).len == (sizeof(s2)-1) &&      \
+     ngx_strncmp((s1).data, (s2), (sizeof(s2)-1)) == 0)
+
+// s1: ngx_str_t, s2: string literal
+// true if they're equal ignoring case, false otherwise
+#define STR_CASE_EQ_LITERAL(s1, s2)     \
+    ((s1).len == (sizeof(s2)-1) &&      \
+     ngx_strncasecmp((s1).data, (       \
+       reinterpret_cast<u_char*>(       \
+         const_cast<char*>(s2))),       \
+       (sizeof(s2)-1)) == 0)
+
+// Allocate memory out of the pool for the string piece, and copy the contents
+// over.  Returns NULL if we can't get memory.
+char* string_piece_to_pool_string(ngx_pool_t* pool, StringPiece sp);
+
+enum PreserveCachingHeaders {
+  kPreserveAllCachingHeaders,  // Cache-Control, ETag, Last-Modified, etc
+  kPreserveOnlyCacheControl,   // Only Cache-Control.
+  kDontPreserveHeaders,
+};
+
+typedef struct {
+  NgxBaseFetch* base_fetch;
+
+  ngx_http_request_t* r;
+
+  bool html_rewrite;
+  bool in_place;
+
+  PreserveCachingHeaders preserve_caching_headers;
+
+  // for html rewrite
+  ProxyFetch* proxy_fetch;
+  GzipInflater* inflater_;
+
+  // for in place resource
+  RewriteDriver* driver;
+  InPlaceResourceRecorder* recorder;
+  ResponseHeaders* ipro_response_headers;
+
+  // We need to remember the URL here as well since we may modify what NGX
+  // gets by stripping our special query params and honoring X-Forwarded-Proto.
+  GoogleString url_string;
+
+  // We need to remember if the upstream had headers_out->location set, because
+  // we should mirror that when we write it back. nginx may absolutify
+  // Location: headers that start with '/' without regarding X-Forwarded-Proto.
+  bool location_field_set;
+  bool psol_vary_accept_only;
+  bool follow_flushes;
+} ps_request_ctx_t;
+
+ps_request_ctx_t* ps_get_request_context(ngx_http_request_t* r);
+
+void copy_request_headers_from_ngx(const ngx_http_request_t* r,
+                                   RequestHeaders* headers);
+
+void copy_response_headers_from_ngx(const ngx_http_request_t* r,
+                                    ResponseHeaders* headers);
+
+ngx_int_t copy_response_headers_to_ngx(
+    ngx_http_request_t* r,
+    const ResponseHeaders& pagespeed_headers,
+    PreserveCachingHeaders preserve_caching_headers);
+
+StringPiece ps_determine_host(ngx_http_request_t* r);
+
+namespace ps_base_fetch {
+
+ngx_int_t ps_base_fetch_handler(ngx_http_request_t* r);
+
+}  // namespace ps_base_fetch
+
+}  // namespace net_instaweb
+
+#endif  // NGX_PAGESPEED_H_

ngx_pagespeed-latest-beta/src/ngx_rewrite_driver_factory.cc

@@ -0,0 +1,288 @@
+/*
+ * Copyright 2012 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+
+#include "ngx_rewrite_driver_factory.h"
+
+#include <cstdio>
+
+#include "log_message_handler.h"
+#include "ngx_message_handler.h"
+#include "ngx_rewrite_options.h"
+#include "ngx_server_context.h"
+#include "ngx_url_async_fetcher.h"
+
+#include "net/instaweb/http/public/rate_controller.h"
+#include "net/instaweb/http/public/rate_controlling_url_async_fetcher.h"
+#include "net/instaweb/http/public/wget_url_fetcher.h"
+#include "net/instaweb/rewriter/public/rewrite_driver.h"
+#include "net/instaweb/rewriter/public/rewrite_driver_factory.h"
+#include "net/instaweb/rewriter/public/server_context.h"
+#include "net/instaweb/util/public/property_cache.h"
+#include "pagespeed/kernel/base/google_message_handler.h"
+#include "pagespeed/kernel/base/null_shared_mem.h"
+#include "pagespeed/kernel/base/posix_timer.h"
+#include "pagespeed/kernel/base/stdio_file_system.h"
+#include "pagespeed/kernel/base/string.h"
+#include "pagespeed/kernel/base/string_util.h"
+#include "pagespeed/kernel/base/thread_system.h"
+#include "pagespeed/kernel/http/content_type.h"
+#include "pagespeed/kernel/sharedmem/shared_circular_buffer.h"
+#include "pagespeed/kernel/sharedmem/shared_mem_statistics.h"
+#include "pagespeed/kernel/thread/pthread_shared_mem.h"
+#include "pagespeed/kernel/thread/scheduler_thread.h"
+#include "pagespeed/kernel/thread/slow_worker.h"
+#include "pagespeed/system/in_place_resource_recorder.h"
+#include "pagespeed/system/serf_url_async_fetcher.h"
+#include "pagespeed/system/system_caches.h"
+#include "pagespeed/system/system_rewrite_options.h"
+
+namespace net_instaweb {
+
+class FileSystem;
+class Hasher;
+class MessageHandler;
+class Statistics;
+class Timer;
+class UrlAsyncFetcher;
+class UrlFetcher;
+class Writer;
+
+class SharedCircularBuffer;
+
+NgxRewriteDriverFactory::NgxRewriteDriverFactory(
+    const ProcessContext& process_context,
+    SystemThreadSystem* system_thread_system, StringPiece hostname, int port)
+    : SystemRewriteDriverFactory(process_context, system_thread_system,
+        NULL /* default shared memory runtime */, hostname, port),
+      threads_started_(false),
+      ngx_message_handler_(
+          new NgxMessageHandler(timer(), thread_system()->NewMutex())),
+      ngx_html_parse_message_handler_(
+          new NgxMessageHandler(timer(), thread_system()->NewMutex())),
+      log_(NULL),
+      resolver_timeout_(NGX_CONF_UNSET_MSEC),
+      use_native_fetcher_(false),
+      // 100 Aligns to nginx's server-side default.
+      native_fetcher_max_keepalive_requests_(100),
+      ngx_shared_circular_buffer_(NULL),
+      hostname_(hostname.as_string()),
+      port_(port),
+      process_script_variables_mode_(ProcessScriptVariablesMode::kOff),
+      process_script_variables_set_(false),
+      shut_down_(false) {
+  InitializeDefaultOptions();
+  default_options()->set_beacon_url("/ngx_pagespeed_beacon");
+  SystemRewriteOptions* system_options = dynamic_cast<SystemRewriteOptions*>(
+      default_options());
+  system_options->set_file_cache_clean_inode_limit(500000);
+  system_options->set_avoid_renaming_introspective_javascript(true);
+  set_message_handler(ngx_message_handler_);
+  set_html_parse_message_handler(ngx_html_parse_message_handler_);
+}
+
+NgxRewriteDriverFactory::~NgxRewriteDriverFactory() {
+  ShutDown();
+  ngx_shared_circular_buffer_ = NULL;
+  STLDeleteElements(&uninitialized_server_contexts_);
+}
+
+Hasher* NgxRewriteDriverFactory::NewHasher() {
+  return new MD5Hasher;
+}
+
+UrlAsyncFetcher* NgxRewriteDriverFactory::AllocateFetcher(
+    SystemRewriteOptions* config) {
+  if (use_native_fetcher_) {
+    NgxUrlAsyncFetcher* fetcher = new NgxUrlAsyncFetcher(
+        config->fetcher_proxy().c_str(),
+        log_,
+        resolver_timeout_,
+        config->blocking_fetch_timeout_ms(),
+        resolver_,
+        native_fetcher_max_keepalive_requests_,
+        thread_system(),
+        message_handler());
+    ngx_url_async_fetchers_.push_back(fetcher);
+    return fetcher;
+  } else {
+    return SystemRewriteDriverFactory::AllocateFetcher(config);
+  }
+}
+
+MessageHandler* NgxRewriteDriverFactory::DefaultHtmlParseMessageHandler() {
+  return ngx_html_parse_message_handler_;
+}
+
+MessageHandler* NgxRewriteDriverFactory::DefaultMessageHandler() {
+  return ngx_message_handler_;
+}
+
+FileSystem* NgxRewriteDriverFactory::DefaultFileSystem() {
+  return new StdioFileSystem();
+}
+
+Timer* NgxRewriteDriverFactory::DefaultTimer() {
+  return new PosixTimer;
+}
+
+NamedLockManager* NgxRewriteDriverFactory::DefaultLockManager() {
+  CHECK(false);
+  return NULL;
+}
+
+RewriteOptions* NgxRewriteDriverFactory::NewRewriteOptions() {
+  NgxRewriteOptions* options = new NgxRewriteOptions(thread_system());
+  // TODO(jefftk): figure out why using SetDefaultRewriteLevel like
+  // mod_pagespeed does in mod_instaweb.cc:create_dir_config() isn't enough here
+  // -- if you use that instead then ngx_pagespeed doesn't actually end up
+  // defaulting CoreFilters.
+  // See: https://github.com/pagespeed/ngx_pagespeed/issues/1190
+  options->SetRewriteLevel(RewriteOptions::kCoreFilters);
+  return options;
+}
+
+RewriteOptions* NgxRewriteDriverFactory::NewRewriteOptionsForQuery() {
+  return new NgxRewriteOptions(thread_system());
+}
+
+bool NgxRewriteDriverFactory::CheckResolver() {
+  if (use_native_fetcher_ && resolver_ == NULL) {
+    return false;
+  }
+  return true;
+}
+
+NgxServerContext* NgxRewriteDriverFactory::MakeNgxServerContext(
+    StringPiece hostname, int port) {
+  NgxServerContext* server_context = new NgxServerContext(this, hostname, port);
+  uninitialized_server_contexts_.insert(server_context);
+  return server_context;
+}
+
+ServerContext* NgxRewriteDriverFactory::NewDecodingServerContext() {
+  ServerContext* sc = new NgxServerContext(this, hostname_, port_);
+  InitStubDecodingServerContext(sc);
+  return sc;
+}
+
+ServerContext* NgxRewriteDriverFactory::NewServerContext() {
+  LOG(DFATAL) << "MakeNgxServerContext should be used instead";
+  return NULL;
+}
+
+void NgxRewriteDriverFactory::ShutDown() {
+  if (!shut_down_) {
+    shut_down_ = true;
+    SystemRewriteDriverFactory::ShutDown();
+  }
+}
+
+void NgxRewriteDriverFactory::ShutDownMessageHandlers() {
+  ngx_message_handler_->set_buffer(NULL);
+  ngx_html_parse_message_handler_->set_buffer(NULL);
+  for (NgxMessageHandlerSet::iterator p =
+           server_context_message_handlers_.begin();
+       p != server_context_message_handlers_.end(); ++p) {
+    (*p)->set_buffer(NULL);
+  }
+  server_context_message_handlers_.clear();
+}
+
+void NgxRewriteDriverFactory::StartThreads() {
+  if (threads_started_) {
+    return;
+  }
+  // TODO(jefftk): use a native nginx timer instead of running our own thread.
+  // See issue #111.
+  SchedulerThread* thread = new SchedulerThread(thread_system(), scheduler());
+  bool ok = thread->Start();
+  CHECK(ok) << "Unable to start scheduler thread";
+  defer_cleanup(thread->MakeDeleter());
+  threads_started_ = true;
+}
+
+void NgxRewriteDriverFactory::SetMainConf(NgxRewriteOptions* main_options) {
+  // Propagate process-scope options from the copy we had during nginx option
+  // parsing to our own.
+  if (main_options != NULL) {
+    default_options()->MergeOnlyProcessScopeOptions(*main_options);
+  }
+}
+
+void NgxRewriteDriverFactory::LoggingInit(
+    ngx_log_t* log, bool may_install_crash_handler) {
+  log_ = log;
+  net_instaweb::log_message_handler::Install(log);
+  if (may_install_crash_handler && install_crash_handler()) {
+    NgxMessageHandler::InstallCrashHandler(log);
+  }
+  ngx_message_handler_->set_log(log);
+  ngx_html_parse_message_handler_->set_log(log);
+}
+
+void NgxRewriteDriverFactory::SetCircularBuffer(
+    SharedCircularBuffer* buffer) {
+  ngx_shared_circular_buffer_ = buffer;
+  ngx_message_handler_->set_buffer(buffer);
+  ngx_html_parse_message_handler_->set_buffer(buffer);
+}
+
+void NgxRewriteDriverFactory::SetServerContextMessageHandler(
+    ServerContext* server_context, ngx_log_t* log) {
+  NgxMessageHandler* handler = new NgxMessageHandler(
+      timer(), thread_system()->NewMutex());
+  handler->set_log(log);
+  // The ngx_shared_circular_buffer_ will be NULL if MessageBufferSize hasn't
+  // been raised from its default of 0.
+  handler->set_buffer(ngx_shared_circular_buffer_);
+  server_context_message_handlers_.insert(handler);
+  defer_cleanup(new Deleter<NgxMessageHandler>(handler));
+  server_context->set_message_handler(handler);
+}
+
+void NgxRewriteDriverFactory::InitStats(Statistics* statistics) {
+  // Init standard PSOL stats.
+  SystemRewriteDriverFactory::InitStats(statistics);
+  RewriteDriverFactory::InitStats(statistics);
+  RateController::InitStats(statistics);
+
+  // Init Ngx-specific stats.
+  NgxServerContext::InitStats(statistics);
+  InPlaceResourceRecorder::InitStats(statistics);
+}
+
+void NgxRewriteDriverFactory::PrepareForkedProcess(const char* name) {
+  ngx_pid = ngx_getpid();  // Needed for logging to have the right PIDs.
+  SystemRewriteDriverFactory::PrepareForkedProcess(name);
+}
+
+void NgxRewriteDriverFactory::NameProcess(const char* name) {
+  SystemRewriteDriverFactory::NameProcess(name);
+
+  // Superclass set status with prctl.  Nginx has a helper function for setting
+  // argv[0] as well, so let's use that.  We'll show up as:
+  //
+  //    nginx: pagespeed $name
+
+  char name_for_setproctitle[32];
+  snprintf(name_for_setproctitle, sizeof(name_for_setproctitle),
+           "pagespeed %s", name);
+  ngx_setproctitle(name_for_setproctitle);
+}
+
+}  // namespace net_instaweb

ngx_pagespeed-latest-beta/src/ngx_rewrite_driver_factory.h

@@ -0,0 +1,176 @@
+/*
+ * Copyright 2012 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+
+#ifndef NGX_REWRITE_DRIVER_FACTORY_H_
+#define NGX_REWRITE_DRIVER_FACTORY_H_
+
+extern "C" {
+  #include <ngx_auto_config.h>
+#if (NGX_THREADS)
+  #include <ngx_thread.h>
+#endif
+  #include <ngx_core.h>
+  #include <ngx_http.h>
+  #include <ngx_config.h>
+  #include <ngx_log.h>
+}
+
+#include <set>
+
+#include "pagespeed/kernel/base/md5_hasher.h"
+#include "pagespeed/kernel/base/scoped_ptr.h"
+#include "pagespeed/system/system_rewrite_driver_factory.h"
+
+namespace net_instaweb {
+
+class NgxMessageHandler;
+class NgxRewriteOptions;
+class NgxServerContext;
+class NgxUrlAsyncFetcher;
+class SharedCircularBuffer;
+class SharedMemRefererStatistics;
+class SlowWorker;
+class Statistics;
+class SystemThreadSystem;
+
+enum ProcessScriptVariablesMode {
+  kOff,
+  kLegacyRestricted,
+  kAll
+};
+
+class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
+ public:
+  // We take ownership of the thread system.
+  explicit NgxRewriteDriverFactory(
+      const ProcessContext& process_context,
+      SystemThreadSystem* system_thread_system, StringPiece hostname, int port);
+  virtual ~NgxRewriteDriverFactory();
+  virtual Hasher* NewHasher();
+  virtual UrlAsyncFetcher* AllocateFetcher(SystemRewriteOptions* config);
+  virtual MessageHandler* DefaultHtmlParseMessageHandler();
+  virtual MessageHandler* DefaultMessageHandler();
+  virtual FileSystem* DefaultFileSystem();
+  virtual Timer* DefaultTimer();
+  virtual NamedLockManager* DefaultLockManager();
+  // Create a new RewriteOptions.  In this implementation it will be an
+  // NgxRewriteOptions, and it will have CoreFilters explicitly set.
+  virtual RewriteOptions* NewRewriteOptions();
+  virtual RewriteOptions* NewRewriteOptionsForQuery();
+  virtual ServerContext* NewDecodingServerContext();
+  // Check resolver configured or not.
+  bool CheckResolver();
+
+  // Initializes all the statistics objects created transitively by
+  // NgxRewriteDriverFactory, including nginx-specific and
+  // platform-independent statistics.
+  static void InitStats(Statistics* statistics);
+  NgxServerContext* MakeNgxServerContext(StringPiece hostname, int port);
+  virtual ServerContext* NewServerContext();
+  virtual void ShutDown();
+
+  // Starts pagespeed threads if they've not been started already.  Must be
+  // called after the caller has finished any forking it intends to do.
+  void StartThreads();
+
+  void SetServerContextMessageHandler(ServerContext* server_context,
+                                      ngx_log_t* log);
+
+  NgxMessageHandler* ngx_message_handler() { return ngx_message_handler_; }
+
+  virtual void NonStaticInitStats(Statistics* statistics) {
+    InitStats(statistics);
+  }
+
+  void SetMainConf(NgxRewriteOptions* main_conf);
+
+  void set_resolver(ngx_resolver_t* resolver) {
+    resolver_ = resolver;
+  }
+  void set_resolver_timeout(ngx_msec_t resolver_timeout) {
+    resolver_timeout_ = resolver_timeout == NGX_CONF_UNSET_MSEC ?
+        1000 : resolver_timeout;
+  }
+  bool use_native_fetcher() {
+    return use_native_fetcher_;
+  }
+  void set_use_native_fetcher(bool x) {
+    use_native_fetcher_ = x;
+  }
+  int native_fetcher_max_keepalive_requests() {
+    return native_fetcher_max_keepalive_requests_;
+  }
+  void set_native_fetcher_max_keepalive_requests(int x) {
+    native_fetcher_max_keepalive_requests_ = x;
+  }
+  ProcessScriptVariablesMode process_script_variables() {
+    return process_script_variables_mode_;
+  }
+
+  void LoggingInit(ngx_log_t* log, bool may_install_crash_handler);
+
+  virtual void ShutDownMessageHandlers();
+
+  virtual void SetCircularBuffer(SharedCircularBuffer* buffer);
+
+  bool SetProcessScriptVariables(ProcessScriptVariablesMode mode) {
+    if (!process_script_variables_set_) {
+      process_script_variables_mode_ = mode;
+      process_script_variables_set_ = true;
+      return true;
+    }
+    return false;
+  }
+
+  virtual void PrepareForkedProcess(const char* name);
+
+  virtual void NameProcess(const char* name);
+
+ private:
+  Timer* timer_;
+
+  bool threads_started_;
+  NgxMessageHandler* ngx_message_handler_;
+  NgxMessageHandler* ngx_html_parse_message_handler_;
+
+  std::vector<NgxUrlAsyncFetcher*> ngx_url_async_fetchers_;
+  ngx_log_t* log_;
+  ngx_msec_t resolver_timeout_;
+  ngx_resolver_t* resolver_;
+  bool use_native_fetcher_;
+  int native_fetcher_max_keepalive_requests_;
+
+  typedef std::set<NgxMessageHandler*> NgxMessageHandlerSet;
+  NgxMessageHandlerSet server_context_message_handlers_;
+
+  // Owned by the superclass.
+  // TODO(jefftk): merge the nginx and apache ways of doing this.
+  SharedCircularBuffer* ngx_shared_circular_buffer_;
+
+  GoogleString hostname_;
+  int port_;
+  ProcessScriptVariablesMode process_script_variables_mode_;
+  bool process_script_variables_set_;
+  bool shut_down_;
+
+  DISALLOW_COPY_AND_ASSIGN(NgxRewriteDriverFactory);
+};
+
+}  // namespace net_instaweb
+
+#endif  // NGX_REWRITE_DRIVER_FACTORY_H_

ngx_pagespeed-latest-beta/src/ngx_rewrite_options.cc

@@ -0,0 +1,552 @@
+/*
+ * Copyright 2012 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+
+#include "ngx_rewrite_options.h"
+
+extern "C" {
+  #include <ngx_config.h>
+  #include <ngx_core.h>
+  #include <ngx_http.h>
+}
+
+#include "ngx_pagespeed.h"
+#include "ngx_rewrite_driver_factory.h"
+
+#include "net/instaweb/public/version.h"
+#include "net/instaweb/rewriter/public/file_load_policy.h"
+#include "net/instaweb/rewriter/public/rewrite_options.h"
+#include "pagespeed/kernel/base/message_handler.h"
+#include "pagespeed/kernel/base/timer.h"
+#include "pagespeed/system/system_caches.h"
+
+namespace net_instaweb {
+
+namespace {
+
+const char kStatisticsPath[] = "StatisticsPath";
+const char kGlobalStatisticsPath[] = "GlobalStatisticsPath";
+const char kConsolePath[] = "ConsolePath";
+const char kMessagesPath[] = "MessagesPath";
+const char kAdminPath[] = "AdminPath";
+const char kGlobalAdminPath[] = "GlobalAdminPath";
+
+// These options are copied from mod_instaweb.cc, where APACHE_CONFIG_OPTIONX
+// indicates that they can not be set at the directory/location level. They set
+// options in the RewriteDriverFactory, so they're entirely global and do not
+// appear in RewriteOptions.  They are not alphabetized on purpose, but rather
+// left in the same order as in mod_instaweb.cc in case we end up needing to
+// compare.
+// TODO(oschaaf): this duplication is a short term solution.
+const char* const server_only_options[] = {
+  "FetcherTimeoutMs",
+  "FetchProxy",
+  "ForceCaching",
+  "GeneratedFilePrefix",
+  "ImgMaxRewritesAtOnce",
+  "InheritVHostConfig",
+  "InstallCrashHandler",
+  "MessageBufferSize",
+  "NumRewriteThreads",
+  "NumExpensiveRewriteThreads",
+  "StaticAssetPrefix",
+  "TrackOriginalContentLength",
+  "UsePerVHostStatistics",  // TODO(anupama): What to do about "No longer used"
+  "BlockingRewriteRefererUrls",
+  "CreateSharedMemoryMetadataCache",
+  "LoadFromFile",
+  "LoadFromFileMatch",
+  "LoadFromFileRule",
+  "LoadFromFileRuleMatch",
+  "UseNativeFetcher",
+  "NativeFetcherMaxKeepaliveRequests"
+};
+
+// Options that can only be used in the main (http) option scope.
+const char* const main_only_options[] = {
+  "UseNativeFetcher",
+  "NativeFetcherMaxKeepaliveRequests"
+};
+
+}  // namespace
+
+RewriteOptions::Properties* NgxRewriteOptions::ngx_properties_ = NULL;
+
+NgxRewriteOptions::NgxRewriteOptions(const StringPiece& description,
+                                     ThreadSystem* thread_system)
+    : SystemRewriteOptions(description, thread_system) {
+  Init();
+}
+
+NgxRewriteOptions::NgxRewriteOptions(ThreadSystem* thread_system)
+    : SystemRewriteOptions(thread_system) {
+  Init();
+}
+
+void NgxRewriteOptions::Init() {
+  DCHECK(ngx_properties_ != NULL)
+      << "Call NgxRewriteOptions::Initialize() before construction";
+  clear_inherited_scripts_ = false;
+  InitializeOptions(ngx_properties_);
+}
+
+void NgxRewriteOptions::AddProperties() {
+  // Nginx-specific options.
+  add_ngx_option(
+      "", &NgxRewriteOptions::statistics_path_, "nsp", kStatisticsPath,
+      kServerScope, "Set the statistics path. Ex: /ngx_pagespeed_statistics",
+      false);
+  add_ngx_option(
+      "", &NgxRewriteOptions::global_statistics_path_, "ngsp",
+      kGlobalStatisticsPath, kProcessScopeStrict,
+      "Set the global statistics path. Ex: /ngx_pagespeed_global_statistics",
+      false);
+  add_ngx_option(
+      "", &NgxRewriteOptions::console_path_, "ncp", kConsolePath, kServerScope,
+      "Set the console path. Ex: /pagespeed_console", false);
+  add_ngx_option(
+      "", &NgxRewriteOptions::messages_path_, "nmp", kMessagesPath,
+      kServerScope, "Set the messages path.  Ex: /ngx_pagespeed_message",
+      false);
+  add_ngx_option(
+      "", &NgxRewriteOptions::admin_path_, "nap", kAdminPath,
+      kServerScope, "Set the admin path.  Ex: /pagespeed_admin", false);
+  add_ngx_option(
+      "", &NgxRewriteOptions::global_admin_path_, "ngap", kGlobalAdminPath,
+      kProcessScopeStrict,
+      "Set the global admin path.  Ex: /pagespeed_global_admin",
+      false);
+
+  MergeSubclassProperties(ngx_properties_);
+
+  // Default properties are global but to set them the current API requires
+  // a RewriteOptions instance and we're in a static method.
+  NgxRewriteOptions dummy_config(NULL);
+  dummy_config.set_default_x_header_value(kModPagespeedVersion);
+}
+
+void NgxRewriteOptions::Initialize() {
+  if (Properties::Initialize(&ngx_properties_)) {
+    SystemRewriteOptions::Initialize();
+    AddProperties();
+  }
+}
+
+void NgxRewriteOptions::Terminate() {
+  if (Properties::Terminate(&ngx_properties_)) {
+    SystemRewriteOptions::Terminate();
+  }
+}
+
+bool NgxRewriteOptions::IsDirective(StringPiece config_directive,
+                                    StringPiece compare_directive) {
+  return StringCaseEqual(config_directive, compare_directive);
+}
+
+RewriteOptions::OptionScope NgxRewriteOptions::GetOptionScope(
+    StringPiece option_name) {
+  ngx_uint_t i;
+  ngx_uint_t size = sizeof(main_only_options) / sizeof(char*);
+  for (i = 0; i < size; i++) {
+    if (StringCaseEqual(main_only_options[i], option_name)) {
+      return kProcessScopeStrict;
+    }
+  }
+
+  size = sizeof(server_only_options) / sizeof(char*);
+  for (i = 0; i < size; i++) {
+    if (StringCaseEqual(server_only_options[i], option_name)) {
+      return kServerScope;
+    }
+  }
+
+  // This could be made more efficient if RewriteOptions provided a map allowing
+  // access of options by their name. It's not too much of a worry at present
+  // since this is just during initialization.
+  for (OptionBaseVector::const_iterator it = all_options().begin();
+       it != all_options().end(); ++it) {
+    RewriteOptions::OptionBase* option = *it;
+    if (option->option_name() == option_name) {
+      // We treat kLegacyProcessScope as kProcessScopeStrict, failing to start
+      // if an option is out of place.
+      return option->scope() == kLegacyProcessScope ? kProcessScopeStrict
+                                                    : option->scope();
+    }
+  }
+  return kDirectoryScope;
+}
+
+RewriteOptions::OptionSettingResult NgxRewriteOptions::ParseAndSetOptions0(
+    StringPiece directive, GoogleString* msg, MessageHandler* handler) {
+  if (IsDirective(directive, "on")) {
+    set_enabled(RewriteOptions::kEnabledOn);
+  } else if (IsDirective(directive, "off")) {
+    set_enabled(RewriteOptions::kEnabledOff);
+  } else if (IsDirective(directive, "unplugged")) {
+    set_enabled(RewriteOptions::kEnabledUnplugged);
+  } else {
+    return RewriteOptions::kOptionNameUnknown;
+  }
+  return RewriteOptions::kOptionOk;
+}
+
+RewriteOptions::OptionSettingResult
+    NgxRewriteOptions::ParseAndSetOptionFromName1(
+        StringPiece name, StringPiece arg,
+        GoogleString* msg, MessageHandler* handler) {
+  // FileCachePath needs error checking.
+  if (StringCaseEqual(name, kFileCachePath)) {
+    if (!StringCaseStartsWith(arg, "/")) {
+      *msg = "must start with a slash";
+      return RewriteOptions::kOptionValueInvalid;
+    }
+  }
+
+  return SystemRewriteOptions::ParseAndSetOptionFromName1(
+      name, arg, msg, handler);
+}
+
+template <class DriverFactoryT>
+RewriteOptions::OptionSettingResult ParseAndSetOptionHelper(
+    StringPiece option_value,
+    DriverFactoryT* driver_factory,
+    void (DriverFactoryT::*set_option_method)(bool)) {
+  bool parsed_value;
+  if (StringCaseEqual(option_value, "on") ||
+      StringCaseEqual(option_value, "true")) {
+    parsed_value = true;
+  } else if (StringCaseEqual(option_value, "off") ||
+             StringCaseEqual(option_value, "false")) {
+    parsed_value = false;
+  } else {
+    return RewriteOptions::kOptionValueInvalid;
+  }
+
+  (driver_factory->*set_option_method)(parsed_value);
+  return RewriteOptions::kOptionOk;
+}
+
+namespace {
+
+const char* ps_error_string_for_option(
+    ngx_pool_t* pool, StringPiece directive, StringPiece warning) {
+  GoogleString msg =
+      StrCat("\"", directive, "\" ", warning);
+  char* s = string_piece_to_pool_string(pool, msg);
+  if (s == NULL) {
+    return "failed to allocate memory";
+  }
+  return s;
+}
+
+}  // namespace
+
+// Very similar to apache/mod_instaweb::ParseDirective.
+const char* NgxRewriteOptions::ParseAndSetOptions(
+    StringPiece* args, int n_args, ngx_pool_t* pool, MessageHandler* handler,
+    NgxRewriteDriverFactory* driver_factory,
+    RewriteOptions::OptionScope scope, ngx_conf_t* cf,
+    ProcessScriptVariablesMode script_mode) {
+  CHECK_GE(n_args, 1);
+
+  StringPiece directive = args[0];
+
+  // Remove initial "ModPagespeed" if there is one.
+  StringPiece mod_pagespeed("ModPagespeed");
+  if (StringCaseStartsWith(directive, mod_pagespeed)) {
+    directive.remove_prefix(mod_pagespeed.size());
+  }
+
+  if (GetOptionScope(directive) > scope) {
+    return ps_error_string_for_option(
+        pool, directive, "cannot be set at this scope.");
+  }
+
+  bool compile_scripts = false;
+
+  if (script_mode != ProcessScriptVariablesMode::kOff) {
+    // In the old mode we only allowed a few, so restrict to those.
+    compile_scripts =
+        StringCaseStartsWith(directive, "LoadFromFile") ||
+        StringCaseEqual(directive, "EnableFilters") ||
+        StringCaseEqual(directive, "DisableFilters") ||
+        StringCaseEqual(directive, "DownstreamCachePurgeLocationPrefix") ||
+        StringCaseEqual(directive, "DownstreamCachePurgeMethod") ||
+        StringCaseEqual(directive,
+                        "DownstreamCacheRewrittenPercentageThreshold") ||
+        StringCaseEqual(directive, "ShardDomain");
+    // In the new behaviour we also allow scripting of query- and directory-
+    // scoped options.
+    compile_scripts |=
+        script_mode == ProcessScriptVariablesMode::kAll &&
+        (GetOptionScope(directive) <= RewriteOptions::kDirectoryScope ||
+         (StringCaseEqual(directive, "Allow") ||
+          StringCaseEqual(directive, "BlockingRewriteRefererUrls") ||
+          StringCaseEqual(directive, "Disallow") ||
+          StringCaseEqual(directive, "DistributableFilters") ||
+          StringCaseEqual(directive, "Domain") ||
+          StringCaseEqual(directive, "ExperimentVariable") ||
+          StringCaseEqual(directive, "ExperimentSpec") ||
+          StringCaseEqual(directive, "ForbidFilters") ||
+          StringCaseEqual(directive, "RetainComment") ||
+          StringCaseEqual(directive, "CustomFetchHeader") ||
+          StringCaseEqual(directive, "MapOriginDomain") ||
+          StringCaseEqual(directive, "MapProxyDomain") ||
+          StringCaseEqual(directive, "MapRewriteDomain") ||
+          StringCaseEqual(directive, "UrlValuedAttribute") ||
+          StringCaseEqual(directive, "Library")));
+  }
+
+  ScriptLine* script_line;
+  script_line = NULL;
+
+  if (n_args == 1 && StringCaseEqual(directive, "ClearInheritedScripts")) {
+    clear_inherited_scripts_ = true;
+    return NGX_CONF_OK;
+  }
+
+  if (compile_scripts) {
+    CHECK(cf != NULL);
+    int i;
+    // Skip the first arg which is always 'pagespeed'
+    for (i = 1; i < n_args; i++) {
+      ngx_str_t script_source;
+
+      script_source.len = args[i].as_string().length();
+      std::string tmp = args[i].as_string();
+      script_source.data = reinterpret_cast<u_char*>(
+          const_cast<char*>(tmp.c_str()));
+
+      if (ngx_http_script_variables_count(&script_source) > 0) {
+        ngx_http_script_compile_t* sc =
+            reinterpret_cast<ngx_http_script_compile_t*>(
+                ngx_pcalloc(cf->pool, sizeof(ngx_http_script_compile_t)));
+        sc->cf = cf;
+        sc->source = &script_source;
+        sc->lengths = reinterpret_cast<ngx_array_t**>(
+            ngx_pcalloc(cf->pool, sizeof(ngx_array_t*)));
+        sc->values = reinterpret_cast<ngx_array_t**>(
+            ngx_pcalloc(cf->pool, sizeof(ngx_array_t*)));
+        sc->variables = 1;
+        sc->complete_lengths = 1;
+        sc->complete_values = 1;
+        if (ngx_http_script_compile(sc) != NGX_OK) {
+          return ps_error_string_for_option(
+              pool, directive, "Failed to compile script variables");
+        } else {
+          if (script_line == NULL) {
+            script_line = new ScriptLine(args, n_args, scope);
+          }
+          script_line->AddScriptAndArgIndex(sc, i);
+        }
+      }
+    }
+
+    if (script_line != NULL) {
+      script_lines_.push_back(RefCountedPtr<ScriptLine>(script_line));
+      // We have found script variables in the current configuration line, and
+      // prepared the associated rewriteoptions for that.
+      // We will defer parsing, validation and processing of this line to
+      // request time. That means we are done handling this configuration line.
+      return NGX_CONF_OK;
+    }
+  }
+
+  GoogleString msg;
+  OptionSettingResult result;
+  if (n_args == 1) {
+    result = ParseAndSetOptions0(directive, &msg, handler);
+  } else if (n_args == 2) {
+    StringPiece arg = args[1];
+    if (IsDirective(directive, "UseNativeFetcher")) {
+      result = ParseAndSetOptionHelper<NgxRewriteDriverFactory>(
+          arg, driver_factory,
+          &NgxRewriteDriverFactory::set_use_native_fetcher);
+    } else if (IsDirective(directive, "NativeFetcherMaxKeepaliveRequests")) {
+      int max_keepalive_requests;
+      if (StringToInt(arg, &max_keepalive_requests) &&
+          max_keepalive_requests > 0) {
+        driver_factory->set_native_fetcher_max_keepalive_requests(
+            max_keepalive_requests);
+        result = RewriteOptions::kOptionOk;
+      } else {
+        result = RewriteOptions::kOptionValueInvalid;
+      }
+    } else if (StringCaseEqual("ProcessScriptVariables", args[0])) {
+      if (scope == RewriteOptions::kProcessScopeStrict) {
+        ProcessScriptVariablesMode mode;
+        if (StringCaseEqual(arg, "all")) {
+          mode = ProcessScriptVariablesMode::kAll;
+        } else if (StringCaseEqual(arg, "on")) {
+          mode = ProcessScriptVariablesMode::kLegacyRestricted;
+        } else if (StringCaseEqual(arg, "off")) {
+          mode = ProcessScriptVariablesMode::kOff;
+        } else {
+          return const_cast<char*>(
+              "pagespeed ProcessScriptVariables: invalid value");
+        }
+        if (driver_factory->SetProcessScriptVariables(mode)) {
+          result = RewriteOptions::kOptionOk;
+        } else {
+          return const_cast<char*>(
+              "pagespeed ProcessScriptVariables: can only be set once");
+        }
+      } else {
+        return const_cast<char*>(
+            "ProcessScriptVariables is only allowed at the top level");
+      }
+    } else {
+      result = ParseAndSetOptionFromName1(directive, arg, &msg, handler);
+      if (result == RewriteOptions::kOptionNameUnknown) {
+        result = driver_factory->ParseAndSetOption1(
+            directive,
+            arg,
+            scope >= RewriteOptions::kLegacyProcessScope,
+            &msg,
+            handler);
+      }
+    }
+  } else if (n_args == 3) {
+    result = ParseAndSetOptionFromName2(directive, args[1], args[2],
+                                        &msg, handler);
+    if (result == RewriteOptions::kOptionNameUnknown) {
+      result = driver_factory->ParseAndSetOption2(
+          directive,
+          args[1],
+          args[2],
+          scope >= RewriteOptions::kLegacyProcessScope,
+          &msg,
+          handler);
+    }
+  } else if (n_args == 4) {
+    result = ParseAndSetOptionFromName3(
+        directive, args[1], args[2], args[3], &msg, handler);
+  } else {
+    result = RewriteOptions::kOptionNameUnknown;
+  }
+
+  switch (result) {
+    case RewriteOptions::kOptionOk:
+      return NGX_CONF_OK;
+    case RewriteOptions::kOptionNameUnknown:
+      return ps_error_string_for_option(
+          pool, directive, "not recognized or too many arguments");
+    case RewriteOptions::kOptionValueInvalid: {
+      GoogleString full_directive;
+      for (int i = 0 ; i < n_args ; i++) {
+        StrAppend(&full_directive, i == 0 ? "" : " ", args[i]);
+      }
+      return ps_error_string_for_option(pool, full_directive, msg);
+    }
+  }
+
+  CHECK(false);
+  return NULL;
+}
+
+// Execute all entries in the script_lines vector, and hand the result off to
+// ParseAndSetOptions to obtain the final option values.
+bool NgxRewriteOptions::ExecuteScriptVariables(
+    ngx_http_request_t* r, MessageHandler* handler,
+    NgxRewriteDriverFactory* driver_factory) {
+  bool script_error = false;
+
+  if (script_lines_.size() > 0) {
+    std::vector<RefCountedPtr<ScriptLine> >::iterator it;
+    for (it = script_lines_.begin() ; it != script_lines_.end(); ++it) {
+      ScriptLine* script_line = it->get();
+      StringPiece args[NGX_PAGESPEED_MAX_ARGS];
+      std::vector<ScriptArgIndex*>::iterator cs_it;
+      int i;
+
+      for (i = 0; i < script_line->n_args(); i++) {
+        args[i] = script_line->args()[i];
+      }
+
+      for (cs_it = script_line->data().begin();
+           cs_it != script_line->data().end(); cs_it++) {
+        ngx_http_script_compile_t* script;
+        ngx_array_t* values;
+        ngx_array_t* lengths;
+        ngx_str_t value;
+
+        script = (*cs_it)->script();
+        lengths = *script->lengths;
+        values = *script->values;
+
+        if (ngx_http_script_run(r, &value, lengths->elts, 0, values->elts)
+            == NULL) {
+          handler->Message(kError, "ngx_http_script_run error");
+          script_error = true;
+          break;
+        } else  {
+          args[(*cs_it)->index()] = str_to_string_piece(value);
+        }
+      }
+
+      const char* status = ParseAndSetOptions(args, script_line->n_args(),
+          r->pool, handler, driver_factory, script_line->scope(), NULL /*cf*/,
+          ProcessScriptVariablesMode::kOff);
+
+      if (status != NULL) {
+        script_error = true;
+        handler->Message(kWarning,
+            "Error setting option value from script: '%s'", status);
+        break;
+      }
+    }
+  }
+
+  if (script_error) {
+    handler->Message(kWarning,
+        "Script error(s) in configuration, disabling optimization");
+    set_enabled(RewriteOptions::kEnabledOff);
+    return false;
+  }
+
+  return true;
+}
+
+void NgxRewriteOptions::CopyScriptLinesTo(
+    NgxRewriteOptions* destination) const {
+  destination->script_lines_ = script_lines_;
+}
+
+void NgxRewriteOptions::AppendScriptLinesTo(
+    NgxRewriteOptions* destination) const {
+  destination->script_lines_.insert(destination->script_lines_.end(),
+                                    script_lines_.begin(), script_lines_.end());
+}
+
+NgxRewriteOptions* NgxRewriteOptions::Clone() const {
+  NgxRewriteOptions* options = new NgxRewriteOptions(
+      StrCat("cloned from ", description()), thread_system());
+  this->CopyScriptLinesTo(options);
+  options->Merge(*this);
+  return options;
+}
+
+const NgxRewriteOptions* NgxRewriteOptions::DynamicCast(
+    const RewriteOptions* instance) {
+  return dynamic_cast<const NgxRewriteOptions*>(instance);
+}
+
+NgxRewriteOptions* NgxRewriteOptions::DynamicCast(RewriteOptions* instance) {
+  return dynamic_cast<NgxRewriteOptions*>(instance);
+}
+
+}  // namespace net_instaweb

ngx_pagespeed-latest-beta/src/ngx_rewrite_options.h

@@ -0,0 +1,248 @@
+/*
+ * Copyright 2012 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+
+// Manage configuration for pagespeed.  Compare to ApacheConfig.
+
+#ifndef NGX_REWRITE_OPTIONS_H_
+#define NGX_REWRITE_OPTIONS_H_
+
+extern "C" {
+  #include <ngx_config.h>
+  #include <ngx_core.h>
+  #include <ngx_http.h>
+}
+
+#include <vector>
+
+#include "ngx_rewrite_driver_factory.h"
+
+#include "net/instaweb/rewriter/public/rewrite_options.h"
+#include "pagespeed/kernel/base/message_handler.h"
+#include "pagespeed/kernel/base/ref_counted_ptr.h"
+#include "pagespeed/kernel/base/stl_util.h"          // for STLDeleteElements
+#include "pagespeed/system/system_rewrite_options.h"
+
+#define NGX_PAGESPEED_MAX_ARGS 10
+
+namespace net_instaweb {
+
+class NgxRewriteDriverFactory;
+
+class ScriptArgIndex {
+ public:
+  explicit ScriptArgIndex(ngx_http_script_compile_t* script, int index)
+    : script_(script), index_(index) {
+      CHECK(script != NULL);
+      CHECK(index > 0 && index < NGX_PAGESPEED_MAX_ARGS);
+  }
+
+  virtual ~ScriptArgIndex() {}
+
+  ngx_http_script_compile_t* script() { return script_; }
+  int index() { return index_; }
+
+ private:
+  // Not owned.
+  ngx_http_script_compile_t* script_;
+  int index_;
+};
+
+// Refcounted, because the ScriptArgIndexes inside data_ can be shared between
+// different rewriteoptions.
+class ScriptLine : public RefCounted<ScriptLine> {
+ public:
+  explicit ScriptLine(StringPiece* args, int n_args,
+                      RewriteOptions::OptionScope scope)
+    : n_args_(n_args),
+      scope_(scope) {
+
+      for (int i = 0; i < n_args; i++) {
+        args_[i] = args[i];
+      }
+  }
+
+  virtual ~ScriptLine() {
+    STLDeleteElements(&data_);
+    data_.clear();
+  }
+
+  void AddScriptAndArgIndex(ngx_http_script_compile_t* script,
+                            int script_index) {
+    CHECK(script != NULL);
+    CHECK(script_index <  NGX_PAGESPEED_MAX_ARGS);
+    data_.push_back(new ScriptArgIndex(script, script_index));
+  }
+
+  int n_args() { return n_args_;}
+  StringPiece* args() { return args_;}
+  RewriteOptions::OptionScope scope() { return scope_; }
+  std::vector<ScriptArgIndex*>& data() {
+    return data_;
+  }
+
+ private:
+  StringPiece args_[NGX_PAGESPEED_MAX_ARGS];
+  int n_args_;
+  RewriteOptions::OptionScope scope_;
+  std::vector<ScriptArgIndex*> data_;
+
+  DISALLOW_COPY_AND_ASSIGN(ScriptLine);
+};
+
+class NgxRewriteOptions : public SystemRewriteOptions {
+ public:
+  // See rewrite_options::Initialize and ::Terminate
+  static void Initialize();
+  static void Terminate();
+
+  NgxRewriteOptions(const StringPiece& description,
+                    ThreadSystem* thread_system);
+  explicit NgxRewriteOptions(ThreadSystem* thread_system);
+  virtual ~NgxRewriteOptions() { }
+
+  // args is an array of n_args StringPieces together representing a directive.
+  // For example:
+  //   ["RewriteLevel", "PassThrough"]
+  // or
+  //   ["EnableFilters", "combine_css,extend_cache,rewrite_images"]
+  // or
+  //   ["ShardDomain", "example.com", "s1.example.com,s2.example.com"]
+  // Apply the directive, returning NGX_CONF_OK on success or an error message
+  // on failure.
+  //
+  // pool is a memory pool for allocating error strings.
+  // cf is only required when compile_scripts is true
+  // when compile_scripts is true, the rewrite_options will be prepared
+  // for replacing any script $variables encountered in args. when false,
+  // script variables will be substituted using the prepared rewrite options.
+  const char* ParseAndSetOptions(
+      StringPiece* args, int n_args, ngx_pool_t* pool, MessageHandler* handler,
+      NgxRewriteDriverFactory* driver_factory, OptionScope scope,
+      ngx_conf_t* cf, ProcessScriptVariablesMode script_mode);
+  bool ExecuteScriptVariables(
+      ngx_http_request_t* r, MessageHandler* handler,
+      NgxRewriteDriverFactory* driver_factory);
+  void CopyScriptLinesTo(NgxRewriteOptions* destination) const;
+  void AppendScriptLinesTo(NgxRewriteOptions* destination) const;
+
+  // Make an identical copy of these options and return it.
+  virtual NgxRewriteOptions* Clone() const;
+
+  // Returns a suitably down cast version of 'instance' if it is an instance
+  // of this class, NULL if not.
+  static const NgxRewriteOptions* DynamicCast(const RewriteOptions* instance);
+  static NgxRewriteOptions* DynamicCast(RewriteOptions* instance);
+
+  const GoogleString& statistics_path() const {
+    return statistics_path_.value();
+  }
+  const GoogleString& global_statistics_path() const {
+    return global_statistics_path_.value();
+  }
+  const GoogleString& console_path() const {
+    return console_path_.value();
+  }
+  const GoogleString& messages_path() const {
+    return messages_path_.value();
+  }
+  const GoogleString& admin_path() const {
+    return admin_path_.value();
+  }
+  const GoogleString& global_admin_path() const {
+    return global_admin_path_.value();
+  }
+  const std::vector<RefCountedPtr<ScriptLine> >& script_lines() const {
+    return script_lines_;
+  }
+  const bool& clear_inherited_scripts() const {
+    return clear_inherited_scripts_;
+  }
+
+ private:
+  // Helper methods for ParseAndSetOptions().  Each can:
+  //  - return kOptionNameUnknown and not set msg:
+  //    - directive not handled; continue on with other possible
+  //      interpretations.
+  //  - return kOptionOk and not set msg:
+  //    - directive handled, all's well.
+  //  - return kOptionValueInvalid and set msg:
+  //    - directive handled with an error; return the error to the user.
+  //
+  // msg will be shown to the user on kOptionValueInvalid.  While it would be
+  // nice to always use msg and never use the MessageHandler, some option
+  // parsing code in RewriteOptions expects to write to a MessageHandler.  If
+  // that happens we put a summary on msg so the user sees something, and the
+  // detailed message goes to their log via handler.
+  OptionSettingResult ParseAndSetOptions0(
+      StringPiece directive, GoogleString* msg, MessageHandler* handler);
+
+  virtual OptionSettingResult ParseAndSetOptionFromName1(
+      StringPiece name, StringPiece arg,
+      GoogleString* msg, MessageHandler* handler);
+
+  // We may want to override 2- and 3-argument versions as well in the future,
+  // but they are not needed yet.
+
+  // Keeps the properties added by this subclass.  These are merged into
+  // RewriteOptions::all_properties_ during Initialize().
+  //
+  // RewriteOptions uses static initialization to reduce memory usage and
+  // construction time.  All NgxRewriteOptions instances will have the same
+  // Properties, so we can build the list when we initialize the first one.
+  static Properties* ngx_properties_;
+  static void AddProperties();
+  void Init();
+
+  // Add an option to ngx_properties_
+  template<class OptionClass>
+  static void add_ngx_option(typename OptionClass::ValueType default_value,
+                             OptionClass NgxRewriteOptions::*offset,
+                             const char* id,
+                             StringPiece option_name,
+                             OptionScope scope,
+                             const char* help,
+                             bool safe_to_print) {
+    AddProperty(default_value, offset, id, option_name, scope, help,
+                safe_to_print, ngx_properties_);
+  }
+
+  Option<GoogleString> statistics_path_;
+  Option<GoogleString> global_statistics_path_;
+  Option<GoogleString> console_path_;
+  Option<GoogleString> messages_path_;
+  Option<GoogleString> admin_path_;
+  Option<GoogleString> global_admin_path_;
+
+  bool clear_inherited_scripts_;
+  std::vector<RefCountedPtr<ScriptLine> > script_lines_;
+
+  // Helper for ParseAndSetOptions.  Returns whether the two directives equal,
+  // ignoring case.
+  bool IsDirective(StringPiece config_directive, StringPiece compare_directive);
+
+  // Returns a given option's scope.
+  RewriteOptions::OptionScope GetOptionScope(StringPiece option_name);
+
+  // TODO(jefftk): support fetch proxy in server and location blocks.
+
+  DISALLOW_COPY_AND_ASSIGN(NgxRewriteOptions);
+};
+
+}  // namespace net_instaweb
+
+#endif  // NGX_REWRITE_OPTIONS_H_

ngx_pagespeed-latest-beta/src/ngx_server_context.cc

@@ -0,0 +1,98 @@
+/*
+ * Copyright 2012 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+
+#include "ngx_server_context.h"
+
+extern "C" {
+  #include <ngx_http.h>
+}
+
+#include "ngx_pagespeed.h"
+#include "ngx_message_handler.h"
+#include "ngx_rewrite_driver_factory.h"
+#include "ngx_rewrite_options.h"
+#include "net/instaweb/rewriter/public/rewrite_driver.h"
+#include "pagespeed/system/add_headers_fetcher.h"
+#include "pagespeed/system/loopback_route_fetcher.h"
+#include "pagespeed/system/system_request_context.h"
+
+namespace net_instaweb {
+
+NgxServerContext::NgxServerContext(
+    NgxRewriteDriverFactory* factory, StringPiece hostname, int port)
+    : SystemServerContext(factory, hostname, port),
+      ngx_http2_variable_index_(NGX_ERROR) {
+}
+
+NgxServerContext::~NgxServerContext() { }
+
+NgxRewriteOptions* NgxServerContext::config() {
+  return NgxRewriteOptions::DynamicCast(global_options());
+}
+
+SystemRequestContext* NgxServerContext::NewRequestContext(
+    ngx_http_request_t* r) {
+  // Based on ngx_http_variable_server_port.
+  bool port_set = false;
+  int local_port = 0;
+#if (NGX_HAVE_INET6)
+  if (r->connection->local_sockaddr->sa_family == AF_INET6) {
+    local_port = ntohs(reinterpret_cast<struct sockaddr_in6*>(
+        r->connection->local_sockaddr)->sin6_port);
+    port_set = true;
+  }
+#endif
+  if (!port_set) {
+    local_port = ntohs(reinterpret_cast<struct sockaddr_in*>(
+        r->connection->local_sockaddr)->sin_port);
+  }
+
+  ngx_str_t local_ip;
+  u_char addr[NGX_SOCKADDR_STRLEN];
+  local_ip.len = NGX_SOCKADDR_STRLEN;
+  local_ip.data = addr;
+  ngx_int_t rc = ngx_connection_local_sockaddr(r->connection, &local_ip, 0);
+  if (rc != NGX_OK) {
+    local_ip.len = 0;
+  }
+
+  SystemRequestContext* ctx = new SystemRequestContext(
+      thread_system()->NewMutex(), timer(),
+      ps_determine_host(r), local_port, str_to_string_piece(local_ip));
+
+  // See if http2 is in use.
+  if (ngx_http2_variable_index_ >= 0) {
+    ngx_http_variable_value_t* val =
+        ngx_http_get_indexed_variable(r, ngx_http2_variable_index_);
+    if (val != NULL && val->valid) {
+      StringPiece str_val(reinterpret_cast<char*>(val->data), val->len);
+      if (str_val == "h2" || str_val == "h2c") {
+        ctx->set_using_http2(true);
+      }
+    }
+  }
+
+  return ctx;
+}
+
+GoogleString NgxServerContext::FormatOption(StringPiece option_name,
+                                            StringPiece args) {
+  return StrCat("pagespeed ", option_name, " ", args, ";");
+}
+
+}  // namespace net_instaweb

ngx_pagespeed-latest-beta/src/ngx_server_context.h

@@ -0,0 +1,79 @@
+/*
+ * Copyright 2012 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+
+// Manage pagespeed state across requests.  Compare to ApacheResourceManager.
+
+#ifndef NGX_SERVER_CONTEXT_H_
+#define NGX_SERVER_CONTEXT_H_
+
+#include "ngx_message_handler.h"
+#include "pagespeed/system/system_server_context.h"
+
+extern "C" {
+#include <ngx_http.h>
+}
+
+namespace net_instaweb {
+
+class NgxRewriteDriverFactory;
+class NgxRewriteOptions;
+class SystemRequestContext;
+
+class NgxServerContext : public SystemServerContext {
+ public:
+  NgxServerContext(
+      NgxRewriteDriverFactory* factory, StringPiece hostname, int port);
+  virtual ~NgxServerContext();
+
+  // We don't allow ProxyFetch to fetch HTML via MapProxyDomain. We will call
+  // set_trusted_input() on any ProxyFetches we use to transform internal HTML.
+  virtual bool ProxiesHtml() const { return false; }
+
+  // Call only when you need an NgxRewriteOptions.  If you don't need
+  // nginx-specific behavior, call global_options() instead which doesn't
+  // downcast.
+  NgxRewriteOptions* config();
+
+  NgxRewriteDriverFactory* ngx_rewrite_driver_factory() { return ngx_factory_; }
+  SystemRequestContext* NewRequestContext(ngx_http_request_t* r);
+
+  NgxMessageHandler* ngx_message_handler() {
+    return dynamic_cast<NgxMessageHandler*>(message_handler());
+  }
+
+  virtual GoogleString FormatOption(StringPiece option_name, StringPiece args);
+
+  void set_ngx_http2_variable_index(ngx_int_t idx) {
+    ngx_http2_variable_index_ = idx;
+  }
+
+  ngx_int_t ngx_http2_variable_index() const {
+    return ngx_http2_variable_index_;
+  }
+
+ private:
+  NgxRewriteDriverFactory* ngx_factory_;
+  // what index the "http2" var is, or NGX_ERROR.
+  ngx_int_t ngx_http2_variable_index_;
+
+  DISALLOW_COPY_AND_ASSIGN(NgxServerContext);
+};
+
+}  // namespace net_instaweb
+
+#endif  // NGX_SERVER_CONTEXT_H_

ngx_pagespeed-latest-beta/src/ngx_url_async_fetcher.cc

@@ -0,0 +1,299 @@
+/*
+ * Copyright 2012 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: x.dinic@gmail.com (Junmin Xiong)
+
+extern "C" {
+  #include <ngx_http.h>
+  #include <ngx_core.h>
+}
+
+#include "ngx_url_async_fetcher.h"
+#include "ngx_fetch.h"
+
+#include <vector>
+#include <algorithm>
+#include <string>
+#include <list>
+#include <map>
+#include <set>
+
+#include "net/instaweb/http/public/async_fetch.h"
+#include "net/instaweb/http/public/inflating_fetch.h"
+#include "net/instaweb/public/version.h"
+#include "pagespeed/kernel/base/basictypes.h"
+#include "pagespeed/kernel/base/condvar.h"
+#include "pagespeed/kernel/base/message_handler.h"
+#include "pagespeed/kernel/base/pool.h"
+#include "pagespeed/kernel/base/pool_element.h"
+#include "pagespeed/kernel/base/statistics.h"
+#include "pagespeed/kernel/base/string_util.h"
+#include "pagespeed/kernel/base/thread_system.h"
+#include "pagespeed/kernel/base/timer.h"
+#include "pagespeed/kernel/base/writer.h"
+#include "pagespeed/kernel/http/request_headers.h"
+#include "pagespeed/kernel/http/response_headers.h"
+#include "pagespeed/kernel/http/response_headers_parser.h"
+
+namespace net_instaweb {
+
+  NgxUrlAsyncFetcher::NgxUrlAsyncFetcher(const char* proxy,
+                                         ngx_log_t* log,
+                                         ngx_msec_t resolver_timeout,
+                                         ngx_msec_t fetch_timeout,
+                                         ngx_resolver_t* resolver,
+                                         int max_keepalive_requests,
+                                         ThreadSystem* thread_system,
+                                         MessageHandler* handler)
+    : fetchers_count_(0),
+      shutdown_(false),
+      track_original_content_length_(false),
+      byte_count_(0),
+      thread_system_(thread_system),
+      message_handler_(handler),
+      mutex_(NULL),
+      max_keepalive_requests_(max_keepalive_requests),
+      event_connection_(NULL) {
+    resolver_timeout_ = resolver_timeout;
+    fetch_timeout_ = fetch_timeout;
+    ngx_memzero(&proxy_, sizeof(proxy_));
+    if (proxy != NULL && *proxy != '\0') {
+      proxy_.url.data = reinterpret_cast<u_char*>(const_cast<char*>(proxy));
+      proxy_.url.len = ngx_strlen(proxy);
+    }
+    mutex_ = thread_system_->NewMutex();
+    log_ = log;
+    pool_ = NULL;
+    resolver_ = resolver;
+    // If init fails, set shutdown_ so no fetches will be attempted.
+    if (!Init(const_cast<ngx_cycle_t*>(ngx_cycle))) {
+      shutdown_ = true;
+      message_handler_->Message(
+          kError, "NgxUrlAsyncFetcher failed to init, fetching disabled.");
+    }
+  }
+
+  NgxUrlAsyncFetcher::~NgxUrlAsyncFetcher() {
+    DCHECK(shutdown_)  << "Shut down before destructing NgxUrlAsyncFetcher.";
+    message_handler_->Message(
+        kInfo,
+        "Destruct NgxUrlAsyncFetcher with [%d] active fetchers",
+        ApproximateNumActiveFetches());
+
+    CancelActiveFetches();
+    active_fetches_.DeleteAll();
+    NgxConnection::Terminate();
+
+    if (pool_ != NULL) {
+      ngx_destroy_pool(pool_);
+      pool_ = NULL;
+    }
+    if (mutex_ != NULL) {
+      delete mutex_;
+      mutex_ = NULL;
+    }
+  }
+
+
+  bool NgxUrlAsyncFetcher::ParseUrl(ngx_url_t* url, ngx_pool_t* pool) {
+    size_t scheme_offset;
+    u_short port;
+    if (ngx_strncasecmp(url->url.data, reinterpret_cast<u_char*>(
+                                     const_cast<char*>("http://")), 7) == 0) {
+      scheme_offset = 7;
+      port = 80;
+    } else if (ngx_strncasecmp(url->url.data, reinterpret_cast<u_char*>(
+                                    const_cast<char*>("https://")), 8) == 0) {
+      scheme_offset = 8;
+      port = 443;
+    } else {
+      scheme_offset = 0;
+      port = 80;
+    }
+
+    url->url.data += scheme_offset;
+    url->url.len -= scheme_offset;
+    url->default_port = port;
+    // See: http://lxr.evanmiller.org/http/source/core/ngx_inet.c#L875
+    url->no_resolve = 0;
+    url->uri_part = 1;
+
+    if (ngx_parse_url(pool, url) == NGX_OK) {
+      return true;
+    }
+    return false;
+  }
+
+  // If there are still active requests, cancel them.
+  void NgxUrlAsyncFetcher::CancelActiveFetches() {
+    // TODO(oschaaf): this seems tricky, this may end up calling
+    // FetchComplete, modifying the active fetches while we are looping
+    // it
+    for (NgxFetchPool::const_iterator p = active_fetches_.begin(),
+        e = active_fetches_.end(); p != e; ++p) {
+      NgxFetch* fetch = *p;
+      fetch->CallbackDone(false);
+    }
+  }
+
+  // Create the pool for fetcher, create the pipe, add the read event for main
+  // thread. It should be called in the worker process.
+  bool NgxUrlAsyncFetcher::Init(ngx_cycle_t* cycle) {
+    log_ = cycle->log;
+    CHECK(event_connection_ == NULL) << "event connection already set";
+    event_connection_ = new NgxEventConnection(ReadCallback);
+    if (!event_connection_->Init(cycle)) {
+      return false;
+    }
+    if (pool_ == NULL) {
+      pool_ = ngx_create_pool(4096, log_);
+      if (pool_ == NULL) {
+        ngx_log_error(NGX_LOG_ERR, log_, 0,
+            "NgxUrlAsyncFetcher::Init create pool failed");
+        return false;
+      }
+    }
+
+    if (proxy_.url.len == 0) {
+      return true;
+    }
+
+    // TODO(oschaaf): shouldn't we do this earlier? Do we need to clean
+    // up when parsing the url failed?
+    if (!ParseUrl(&proxy_, pool_)) {
+      ngx_log_error(NGX_LOG_ERR, log_, 0,
+          "NgxUrlAsyncFetcher::Init parse proxy[%V] failed", &proxy_.url);
+      return false;
+    }
+    return true;
+  }
+
+  void NgxUrlAsyncFetcher::ShutDown() {
+    shutdown_ = true;
+    if (!pending_fetches_.empty()) {
+      for (Pool<NgxFetch>::iterator p = pending_fetches_.begin(),
+           e = pending_fetches_.end(); p != e; p++) {
+        NgxFetch* fetch = *p;
+        fetch->CallbackDone(false);
+      }
+      pending_fetches_.DeleteAll();
+    }
+
+    if (!active_fetches_.empty()) {
+      for (Pool<NgxFetch>::iterator p = active_fetches_.begin(),
+           e = active_fetches_.end(); p != e; p++) {
+        NgxFetch* fetch = *p;
+        fetch->CallbackDone(false);
+      }
+      active_fetches_.Clear();
+    }
+    if (event_connection_ != NULL) {
+      event_connection_->Shutdown();
+      delete event_connection_;
+      event_connection_ = NULL;
+    }
+  }
+
+  // It's called in the rewrite thread. All the fetches are started at
+  // this function. It will notify the main thread to start the fetch job.
+  void NgxUrlAsyncFetcher::Fetch(const GoogleString& url,
+                                 MessageHandler* message_handler,
+                                 AsyncFetch* async_fetch) {
+    // Don't accept new fetches when shut down. This flow is also entered when
+    // we did not initialize properly in ::Init().
+    if (shutdown_) {
+      async_fetch->Done(false);
+      return;
+    }
+    async_fetch = EnableInflation(async_fetch);
+    NgxFetch* fetch = new NgxFetch(url, async_fetch,
+          message_handler, log_);
+    ScopedMutex lock(mutex_);
+    pending_fetches_.Add(fetch);
+
+    // TODO(oschaaf): thread safety on written vs shutdown.
+    // It is possible that shutdown() is called after writing an event? In that
+    // case, this could (rarely) fail when it shouldn't.
+    bool written = event_connection_->WriteEvent(this);
+    CHECK(written || shutdown_) << "NgxUrlAsyncFetcher: event write failure";
+  }
+
+  // This is the read event which is called in the main thread.
+  // It will do the real work. Add the work event and start the fetch.
+  void NgxUrlAsyncFetcher::ReadCallback(const ps_event_data& data) {
+    std::vector<NgxFetch*> to_start;
+    NgxUrlAsyncFetcher* fetcher = reinterpret_cast<NgxUrlAsyncFetcher*>(
+      data.sender);
+
+    fetcher->mutex_->Lock();
+    fetcher->completed_fetches_.DeleteAll();
+
+    for (Pool<NgxFetch>::iterator p = fetcher->pending_fetches_.begin(),
+             e = fetcher->pending_fetches_.end(); p != e; p++) {
+      NgxFetch* fetch = *p;
+      to_start.push_back(fetch);
+    }
+
+    fetcher->pending_fetches_.Clear();
+    fetcher->mutex_->Unlock();
+
+    for (size_t i = 0; i < to_start.size(); i++) {
+      fetcher->StartFetch(to_start[i]);
+    }
+
+    return;
+  }
+
+  // TODO(oschaaf): return value is ignored.
+  bool NgxUrlAsyncFetcher::StartFetch(NgxFetch* fetch) {
+    mutex_->Lock();
+    active_fetches_.Add(fetch);
+    fetchers_count_++;
+    mutex_->Unlock();
+
+    // Don't initiate the fetch when we are shutting down
+    if (shutdown_) {
+      fetch->CallbackDone(false);
+      return false;
+    }
+
+    bool started = fetch->Start(this);
+
+    if (!started) {
+      message_handler_->Message(kWarning, "Fetch failed to start: %s",
+                                fetch->str_url());
+      fetch->CallbackDone(false);
+    }
+
+    return started;
+  }
+
+  void NgxUrlAsyncFetcher::FetchComplete(NgxFetch* fetch) {
+    ScopedMutex lock(mutex_);
+    byte_count_ += fetch->bytes_received();
+    fetchers_count_--;
+    active_fetches_.Remove(fetch);
+    completed_fetches_.Add(fetch);
+  }
+
+  void NgxUrlAsyncFetcher::PrintActiveFetches(MessageHandler* handler) const {
+    for (NgxFetchPool::const_iterator p = active_fetches_.begin(),
+        e = active_fetches_.end(); p != e; ++p) {
+      NgxFetch* fetch = *p;
+      handler->Message(kInfo, "Active fetch: %s", fetch->str_url());
+    }
+  }
+}  // namespace net_instaweb

ngx_pagespeed-latest-beta/src/ngx_url_async_fetcher.h

@@ -0,0 +1,154 @@
+/*
+ * Copyright 2012 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: x.dinic@gmail.com(Junmin Xiong)
+//
+// Fetch the resources asynchronously in Nginx. The fetcher is called in
+// the rewrite thread.
+//
+// It can communicate with Nginx by pipe. One pipe for one fetcher.
+// When new url fetch comes, Fetcher will add it to the pending queue and
+// notify the Nginx thread to start the Fetch event. All the events are hooked
+// in the main thread's epoll structure.
+
+#ifndef NET_INSTAWEB_NGX_URL_ASYNC_FETCHER_H_
+#define NET_INSTAWEB_NGX_URL_ASYNC_FETCHER_H_
+
+extern "C" {
+  #include <ngx_config.h>
+  #include <ngx_core.h>
+}
+
+#include <vector>
+
+#include "ngx_event_connection.h"
+
+#include "net/instaweb/http/public/url_async_fetcher.h"
+#include "pagespeed/kernel/base/basictypes.h"
+#include "pagespeed/kernel/base/pool.h"
+#include "pagespeed/kernel/base/string.h"
+#include "pagespeed/kernel/base/thread_system.h"
+
+
+namespace net_instaweb {
+
+class AsyncFetch;
+class MessageHandler;
+class Statistics;
+class NgxFetch;
+class Variable;
+
+class NgxUrlAsyncFetcher : public UrlAsyncFetcher {
+ public:
+  NgxUrlAsyncFetcher(
+      const char* proxy, ngx_log_t* log, ngx_msec_t resolver_timeout,
+      ngx_msec_t fetch_timeout, ngx_resolver_t* resolver,
+      int max_keepalive_requests, ThreadSystem* thread_system,
+      MessageHandler* handler);
+
+  ~NgxUrlAsyncFetcher();
+
+  // It should be called in the module init_process callback function. Do some
+  // intializations which can't be done in the master process
+  bool Init(ngx_cycle_t* cycle);
+
+  // shutdown all the fetches.
+  virtual void ShutDown();
+
+  // the read handler in the main thread
+  static void ReadCallback(const ps_event_data& data);
+
+  virtual bool SupportsHttps() const { return false; }
+
+  virtual void Fetch(const GoogleString& url,
+                     MessageHandler* message_handler,
+                     AsyncFetch* callback);
+
+  bool StartFetch(NgxFetch* fetch);
+
+  // Remove the completed fetch from the active fetch set, and put it into a
+  // completed fetch list to be cleaned up.
+  void FetchComplete(NgxFetch* fetch);
+  void PrintActiveFetches(MessageHandler* handler) const;
+
+  // Indicates that it should track the original content length for
+  // fetched resources.
+  bool track_original_content_length() {
+    return track_original_content_length_;
+  }
+  void set_track_original_content_length(bool x) {
+    track_original_content_length_ = x;
+  }
+
+  typedef Pool<NgxFetch> NgxFetchPool;
+
+  // AnyPendingFetches is accurate only at the time of call; this is
+  // used conservatively during shutdown.  It counts fetches that have been
+  // requested by some thread, and can include fetches for which no action
+  // has yet been taken (ie fetches that are not active).
+  virtual bool AnyPendingFetches() {
+    return !active_fetches_.empty();
+  }
+
+  // ApproximateNumActiveFetches can under- or over-count and is used only for
+  // error reporting.
+  int ApproximateNumActiveFetches() {
+    return active_fetches_.size();
+  }
+
+  void CancelActiveFetches();
+
+  // These must be accessed with mutex_ held.
+  bool shutdown() const { return shutdown_; }
+  void set_shutdown(bool s) { shutdown_ = s; }
+
+
+ private:
+  static void TimeoutHandler(ngx_event_t* tev);
+  static bool ParseUrl(ngx_url_t* url, ngx_pool_t* pool);
+  friend class NgxFetch;
+
+  NgxFetchPool active_fetches_;
+  // Add the pending task to this list
+  NgxFetchPool pending_fetches_;
+  NgxFetchPool completed_fetches_;
+  ngx_url_t proxy_;
+
+  int fetchers_count_;
+  bool shutdown_;
+  bool track_original_content_length_;
+  int64 byte_count_;
+  ThreadSystem* thread_system_;
+  MessageHandler* message_handler_;
+  // Protect the member variable in this class
+  // active_fetches, pending_fetches
+  ThreadSystem::CondvarCapableMutex* mutex_;
+
+  ngx_pool_t* pool_;
+  ngx_log_t* log_;
+  ngx_resolver_t* resolver_;
+  int max_keepalive_requests_;
+  ngx_msec_t resolver_timeout_;
+  ngx_msec_t fetch_timeout_;
+
+  NgxEventConnection* event_connection_;
+
+  DISALLOW_COPY_AND_ASSIGN(NgxUrlAsyncFetcher);
+};
+
+}  // namespace net_instaweb
+
+#endif  // NET_INSTAWEB_NGX_URL_ASYNC_FETCHER_H_$