dircache.c File Reference

Cache directories and serve them to clients. More...

#include "core/or/or.h"
#include "app/config/config.h"
#include "app/config/resolve_addr.h"
#include "core/mainloop/connection.h"
#include "core/or/relay.h"
#include "feature/dirauth/dirvote.h"
#include "feature/dirauth/authmode.h"
#include "feature/dirauth/process_descs.h"
#include "feature/dircache/conscache.h"
#include "feature/dircache/consdiffmgr.h"
#include "feature/dircache/dircache.h"
#include "feature/dircache/dirserv.h"
#include "feature/dircommon/directory.h"
#include "feature/dircommon/fp_pair.h"
#include "feature/hs/hs_cache.h"
#include "feature/hs/hs_ident.h"
#include "feature/nodelist/authcert.h"
#include "feature/nodelist/networkstatus.h"
#include "feature/nodelist/routerlist.h"
#include "feature/relay/relay_config.h"
#include "feature/relay/routermode.h"
#include "feature/stats/geoip_stats.h"
#include "feature/stats/rephist.h"
#include "lib/compress/compress.h"
#include "lib/crypt_ops/crypto_format.h"
#include "feature/dircache/cached_dir_st.h"
#include "feature/dircommon/dir_connection_st.h"
#include "feature/nodelist/authority_cert_st.h"
#include "feature/nodelist/networkstatus_st.h"
#include "feature/nodelist/routerinfo_st.h"

Go to the source code of this file.

Data Structures
struct	get_handler_args_t
struct	url_table_ent_t
struct	parsed_consensus_request_t

Macros
#define	DIRCACHE_PRIVATE
#define	MAX_DIR_UL_SIZE   ((1<<24)-1) /* 16MB-1 */
#define	FULL_DIR_CACHE_LIFETIME   (60*60)
#define	RUNNINGROUTERS_CACHE_LIFETIME   (20*60)
#define	DIRPORTFRONTPAGE_CACHE_LIFETIME   (20*60)
#define	NETWORKSTATUS_CACHE_LIFETIME   (5*60)
#define	ROUTERDESC_CACHE_LIFETIME   (30*60)
#define	ROUTERDESC_BY_DIGEST_CACHE_LIFETIME   (48*60*60)
#define	ROBOTS_CACHE_LIFETIME   (24*60*60)
#define	MICRODESC_CACHE_LIFETIME   (48*60*60)
#define	BANDWIDTH_CACHE_LIFETIME   (30*60)
#define	NOT_REASONABLY_LIVE_WARNING_INTERVAL   (60*60)
#define	FALLBACK_COMPRESS_METHOD   ZLIB_METHOD

Functions
STATIC int	parse_http_url (const char *headers, char **url)
static void	write_short_http_response (dir_connection_t *conn, int status, const char *reason_phrase)
static void	write_http_response_header_impl (dir_connection_t *conn, ssize_t length, const char *type, const char *encoding, const char *extra_headers, long cache_lifetime)
static void	write_http_response_headers (dir_connection_t *conn, ssize_t length, compress_method_t method, const char *extra_headers, long cache_lifetime)
static void	write_http_response_header (dir_connection_t *conn, ssize_t length, compress_method_t method, long cache_lifetime)
STATIC unsigned	parse_accept_encoding_header (const char *h)
static int	client_likes_consensus (const struct consensus_cache_entry_t *ent, const char *want_url)
STATIC compression_level_t	choose_compression_level (void)
static int	handle_get_frontpage (dir_connection_t *conn, const get_handler_args_t *args)
static int	handle_get_current_consensus (dir_connection_t *conn, const get_handler_args_t *args)
static int	handle_get_status_vote (dir_connection_t *conn, const get_handler_args_t *args)
static int	handle_get_microdesc (dir_connection_t *conn, const get_handler_args_t *args)
static int	handle_get_descriptor (dir_connection_t *conn, const get_handler_args_t *args)
static int	handle_get_keys (dir_connection_t *conn, const get_handler_args_t *args)
static int	handle_get_robots (dir_connection_t *conn, const get_handler_args_t *args)
static int	handle_get_networkstatus_bridges (dir_connection_t *conn, const get_handler_args_t *args)
static int	handle_get_next_bandwidth (dir_connection_t *conn, const get_handler_args_t *args)
STATIC int	directory_handle_command_get (dir_connection_t *conn, const char *headers, const char *req_body, size_t req_body_len)
static void	warn_consensus_is_not_reasonably_live (const struct consensus_cache_entry_t *consensus, const char *flavor, time_t now, bool is_too_new)
static int	parse_one_diff_hash (uint8_t *digest, const char *hex, const char *location, const char *action)
static int	parse_or_diff_from_header (smartlist_t **digests_out, const char *headers)
static struct consensus_cache_entry_t *	find_best_diff (const smartlist_t *digests, int flav, unsigned compression_methods, compress_method_t *compression_used_out)
static struct consensus_cache_entry_t *	find_best_consensus (int flav, unsigned compression_methods, compress_method_t *compression_used_out)
static compress_method_t	find_best_compression_method (unsigned compression_methods, int stream)
static int	digest_list_contains_best_consensus (consensus_flavor_t flavor, const smartlist_t *digests)
static void	parsed_consensus_request_clear (parsed_consensus_request_t *req)
static int	parse_consensus_request (parsed_consensus_request_t *out, const get_handler_args_t *args)
STATIC int	handle_get_hs_descriptor_v3 (dir_connection_t *conn, const get_handler_args_t *args)
STATIC int	parse_hs_version_from_post (const char *url, const char *prefix, const char **end_pos)
STATIC int	handle_post_hs_descriptor (const char *url, const char *body)
STATIC int	directory_handle_command_post (dir_connection_t *conn, const char *headers, const char *body, size_t body_len)
static void	http_set_address_origin (const char *headers, connection_t *conn)
int	directory_handle_command (dir_connection_t *conn)

Variables
static compress_method_t	srv_meth_pref_precompressed []
static compress_method_t	srv_meth_pref_streaming_compression []
static const url_table_ent_t	url_table []

Detailed Description

Cache directories and serve them to clients.

Definition in file dircache.c.

Macro Definition Documentation

BANDWIDTH_CACHE_LIFETIME

#define BANDWIDTH_CACHE_LIFETIME   (30*60)

Definition at line 61 of file dircache.c.

DIRCACHE_PRIVATE

#define DIRCACHE_PRIVATE

Definition at line 11 of file dircache.c.

DIRPORTFRONTPAGE_CACHE_LIFETIME

#define DIRPORTFRONTPAGE_CACHE_LIFETIME   (20*60)

Definition at line 54 of file dircache.c.

FALLBACK_COMPRESS_METHOD

#define FALLBACK_COMPRESS_METHOD   ZLIB_METHOD

Fallback compression method. The fallback compression method is used in case a client requests a non-compressed document. We only store compressed documents, so we use this compression method to fetch the document and let the spooling system do the streaming decompression.

Definition at line 603 of file dircache.c.

Referenced by find_best_consensus(), and find_best_diff().

FULL_DIR_CACHE_LIFETIME

#define FULL_DIR_CACHE_LIFETIME   (60*60)

HTTP cache control: how long do we tell proxies they can cache each kind of document we serve?

Definition at line 52 of file dircache.c.

Referenced by handle_get_descriptor().

MAX_DIR_UL_SIZE

#define MAX_DIR_UL_SIZE   ((1<<24)-1) /* 16MB-1 */

Maximum size, in bytes, for any directory object that we're accepting as an upload.

Definition at line 48 of file dircache.c.

Referenced by directory_handle_command().

MICRODESC_CACHE_LIFETIME

#define MICRODESC_CACHE_LIFETIME   (48*60*60)

Definition at line 59 of file dircache.c.

NETWORKSTATUS_CACHE_LIFETIME

#define NETWORKSTATUS_CACHE_LIFETIME   (5*60)

Definition at line 55 of file dircache.c.

ROBOTS_CACHE_LIFETIME

#define ROBOTS_CACHE_LIFETIME   (24*60*60)

Definition at line 58 of file dircache.c.

ROUTERDESC_BY_DIGEST_CACHE_LIFETIME

#define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME   (48*60*60)

Definition at line 57 of file dircache.c.

ROUTERDESC_CACHE_LIFETIME

#define ROUTERDESC_CACHE_LIFETIME   (30*60)

Definition at line 56 of file dircache.c.

RUNNINGROUTERS_CACHE_LIFETIME

#define RUNNINGROUTERS_CACHE_LIFETIME   (20*60)

Definition at line 53 of file dircache.c.

Function Documentation

choose_compression_level()

STATIC compression_level_t choose_compression_level

(

void

)

Return the compression level we should use for sending a compressed response of size n_bytes.

Definition at line 300 of file dircache.c.

Referenced by handle_get_descriptor(), handle_get_keys(), handle_get_microdesc(), handle_get_next_bandwidth(), and handle_get_status_vote().

client_likes_consensus()

static int client_likes_consensus ( const struct consensus_cache_entry_t * ent, const char * want_url )

static

Decide whether a client would accept the consensus we have.

Clients can say they only want a consensus if it's signed by more than half the authorities in a list. They pass this list in the url as "...consensus/<b>fpr</b>+<b>fpr</b>+<b>fpr</b>".

fpr may be an abbreviated fingerprint, i.e. only a left substring of the full authority identity digest. (Only strings of even length, i.e. encodings of full bytes, are handled correctly. In the case of an odd number of hex digits the last one is silently ignored.)

Returns 1 if more than half of the requested authorities signed the consensus, 0 otherwise.

Definition at line 260 of file dircache.c.

Referenced by handle_get_current_consensus().

digest_list_contains_best_consensus()

static int digest_list_contains_best_consensus ( consensus_flavor_t flavor, const smartlist_t * digests )

static

Check if any of the digests in digests matches the latest consensus flavor (given in flavor) that we have available.

Definition at line 713 of file dircache.c.

Referenced by handle_get_current_consensus().

directory_handle_command()

int directory_handle_command

(

dir_connection_t *

conn

)

Called when a dirserver receives data on a directory connection; looks for an HTTP request. If the request is complete, remove it from the inbuf, try to process it; otherwise, leave it on the buffer. Return a 0 on success, or -1 on error.

Definition at line 1739 of file dircache.c.

Referenced by connection_dir_process_inbuf().

directory_handle_command_get()

STATIC int directory_handle_command_get	(	dir_connection_t *	conn,
		const char *	headers,
		const char *	req_body,
		size_t	req_body_len )

Helper function: called when a dirserver gets a complete HTTP GET request. Look for a request for a directory or for a rendezvous service descriptor. On finding one, write a response into conn->outbuf. If the request is unrecognized, send a 404. Return 0 if we handled this successfully, or -1 if we need to close the connection.

Definition at line 392 of file dircache.c.

Referenced by directory_handle_command().

directory_handle_command_post()

STATIC int directory_handle_command_post	(	dir_connection_t *	conn,
		const char *	headers,
		const char *	body,
		size_t	body_len )

Helper function: called when a dirserver gets a complete HTTP POST request. Look for an uploaded server descriptor or rendezvous service descriptor. On finding one, process it and write a response into conn->outbuf. If the request is unrecognized, send a

1. Always return 0.

Definition at line 1580 of file dircache.c.

Referenced by directory_handle_command().

find_best_compression_method()

static compress_method_t find_best_compression_method ( unsigned compression_methods, int stream )

static

Try to find the best supported compression method possible from a given compression_methods. Return NO_METHOD if no mutually supported compression method could be found.

Definition at line 687 of file dircache.c.

Referenced by handle_get_current_consensus(), handle_get_descriptor(), handle_get_keys(), handle_get_microdesc(), handle_get_next_bandwidth(), and handle_get_status_vote().

find_best_consensus()

static struct consensus_cache_entry_t * find_best_consensus ( int flav, unsigned compression_methods, compress_method_t * compression_used_out )

static

Lookup the cached consensus document by the flavor found in flav. The preferred set of compression methods should be listed in the compression_methods bitfield. The compression method chosen (if any) is stored in compression_used_out.

Definition at line 652 of file dircache.c.

Referenced by handle_get_current_consensus().

find_best_diff()

static struct consensus_cache_entry_t * find_best_diff ( const smartlist_t * digests, int flav, unsigned compression_methods, compress_method_t * compression_used_out )

static

Try to find the best consensus diff possible in order to serve a client request for a diff from one of the consensuses in digests to the current consensus of flavor flav. The client supports the compression methods listed in the compression_methods bitfield: place the method chosen (if any) into compression_used_out.

Definition at line 613 of file dircache.c.

Referenced by handle_get_current_consensus().

handle_get_current_consensus()

static int handle_get_current_consensus ( dir_connection_t * conn, const get_handler_args_t * args )

static

Helper function for GET /tor/status-vote/current/consensus

Definition at line 850 of file dircache.c.

handle_get_descriptor()

static int handle_get_descriptor ( dir_connection_t * conn, const get_handler_args_t * args )

static

Helper function for GET /tor/{server,extra}/...

Definition at line 1162 of file dircache.c.

handle_get_frontpage()

static int handle_get_frontpage ( dir_connection_t * conn, const get_handler_args_t * args )

static

Helper function for GET / or GET /tor/

Definition at line 481 of file dircache.c.

handle_get_hs_descriptor_v3()

STATIC int handle_get_hs_descriptor_v3	(	dir_connection_t *	conn,
		const get_handler_args_t *	args )

Helper function for GET /tor/hs/3/.... Only for version 3.

Definition at line 1354 of file dircache.c.

handle_get_keys()

static int handle_get_keys ( dir_connection_t * conn, const get_handler_args_t * args )

static

Helper function for GET /tor/keys/...

Definition at line 1257 of file dircache.c.

handle_get_microdesc()

static int handle_get_microdesc ( dir_connection_t * conn, const get_handler_args_t * args )

static

Helper function for GET /tor/micro/d/...

Definition at line 1108 of file dircache.c.

handle_get_networkstatus_bridges()

static int handle_get_networkstatus_bridges ( dir_connection_t * conn, const get_handler_args_t * args )

static

Helper function for GET /tor/networkstatus-bridges

Definition at line 1404 of file dircache.c.

handle_get_next_bandwidth()

static int handle_get_next_bandwidth ( dir_connection_t * conn, const get_handler_args_t * args )

static

Helper function for GET the bandwidth file used for the next vote

Definition at line 1444 of file dircache.c.

handle_get_robots()

static int handle_get_robots ( dir_connection_t * conn, const get_handler_args_t * args )

static

Helper function for GET robots.txt or /tor/robots.txt

Definition at line 1477 of file dircache.c.

handle_get_status_vote()

static int handle_get_status_vote ( dir_connection_t * conn, const get_handler_args_t * args )

static

Helper function for GET /tor/status-vote/{current,next}/...

Definition at line 1021 of file dircache.c.

handle_post_hs_descriptor()

STATIC int handle_post_hs_descriptor	(	const char *	url,
		const char *	body )

Definition at line 1535 of file dircache.c.

http_set_address_origin()

static void http_set_address_origin ( const char * headers, connection_t * conn )

static

If headers indicates that a proxy was involved, then rewrite conn->address to describe our best guess of the address that originated this HTTP request.

Definition at line 1711 of file dircache.c.

Referenced by directory_handle_command().

parse_accept_encoding_header()

STATIC unsigned parse_accept_encoding_header

(

const char *

h

)

Parse the compression methods listed in an Accept-Encoding header h, and convert them to a bitfield where compression method x is supported if and only if 1 << x is set in the bitfield.

Definition at line 224 of file dircache.c.

Referenced by directory_handle_command_get().

parse_consensus_request()

static int parse_consensus_request ( parsed_consensus_request_t * out, const get_handler_args_t * args )

static

Parse the URL and relevant headers of args for a current-consensus request to learn what flavor of consensus we want, what keys it must be signed with, and what diffs we would accept (or demand) instead. Return 0 on success and -1 on failure.

Definition at line 775 of file dircache.c.

Referenced by handle_get_current_consensus().

parse_hs_version_from_post()

STATIC int parse_hs_version_from_post	(	const char *	url,
		const char *	prefix,
		const char **	end_pos )

Definition at line 1499 of file dircache.c.

parse_http_url()

STATIC int parse_http_url	(	const char *	headers,
		char **	url )

Parse an HTTP request string headers of the form

* "\%s [http[s]://]\%s HTTP/1..."
* 

If it's well-formed, strdup the second %s into *url, and nul-terminate it. If the url doesn't start with "/tor/", rewrite it so it does. Return 0. Otherwise, return -1.

Definition at line 72 of file dircache.c.

Referenced by directory_handle_command_get(), and directory_handle_command_post().

parse_one_diff_hash()

static int parse_one_diff_hash ( uint8_t * digest, const char * hex, const char * location, const char * action )

static

Parse a single hex-encoded sha3-256 digest from hex into digest. Return 0 on success. On failure, report that the hash came from location, report that we are taking action with it, and return -1.

Definition at line 556 of file dircache.c.

Referenced by parse_consensus_request(), and parse_or_diff_from_header().

parse_or_diff_from_header()

static int parse_or_diff_from_header ( smartlist_t ** digests_out, const char * headers )

static

If there is an X-Or-Diff-From-Consensus header included in headers, set digest_out to a new smartlist containing every 256-bit hex-encoded digest listed in that header and return 0. Otherwise return -1.

Definition at line 575 of file dircache.c.

Referenced by parse_consensus_request().

parsed_consensus_request_clear()

static void parsed_consensus_request_clear ( parsed_consensus_request_t * req )

static

Remove all data held in req. Do not free req itself, since it is stack-allocated.

Definition at line 756 of file dircache.c.

Referenced by handle_get_current_consensus().

warn_consensus_is_not_reasonably_live()

static void warn_consensus_is_not_reasonably_live ( const struct consensus_cache_entry_t * consensus, const char * flavor, time_t now, bool is_too_new )

static

Warn that the cached consensus consensus of type flavor too new or too old, based on is_too_new, and will not be served to clients. Rate-limit the warning to avoid logging an entry on every request.

Definition at line 509 of file dircache.c.

Referenced by handle_get_current_consensus().

write_http_response_header()

static void write_http_response_header ( dir_connection_t * conn, ssize_t length, compress_method_t method, long cache_lifetime )

static

As write_http_response_headers, but assumes extra_headers is NULL

Definition at line 194 of file dircache.c.

Referenced by handle_get_descriptor(), handle_get_hs_descriptor_v3(), handle_get_keys(), handle_get_microdesc(), handle_get_networkstatus_bridges(), handle_get_next_bandwidth(), handle_get_robots(), and handle_get_status_vote().

write_http_response_header_impl()

static void write_http_response_header_impl ( dir_connection_t * conn, ssize_t length, const char * type, const char * encoding, const char * extra_headers, long cache_lifetime )

static

Write the header for an HTTP/1.0 response onto conn->outbuf, with type as the Content-Type.

If length is nonnegative, it is the Content-Length. If encoding is provided, it is the Content-Encoding. If cache_lifetime is greater than 0, the content may be cached for up to cache_lifetime seconds. Otherwise, the content may not be cached.

Definition at line 129 of file dircache.c.

Referenced by directory_handle_command_post(), handle_get_frontpage(), and write_http_response_headers().

write_http_response_headers()

static void write_http_response_headers ( dir_connection_t * conn, ssize_t length, compress_method_t method, const char * extra_headers, long cache_lifetime )

static

As write_http_response_header_impl, but translates method into encoding

Definition at line 181 of file dircache.c.

Referenced by handle_get_current_consensus(), and write_http_response_header().

write_short_http_response()

static void write_short_http_response ( dir_connection_t * conn, int status, const char * reason_phrase )

static

Create an http response for the client conn out of status and reason_phrase. Write it to conn.

Definition at line 94 of file dircache.c.

Referenced by directory_handle_command_get(), directory_handle_command_post(), handle_get_current_consensus(), handle_get_descriptor(), handle_get_frontpage(), handle_get_hs_descriptor_v3(), handle_get_keys(), handle_get_microdesc(), handle_get_networkstatus_bridges(), handle_get_next_bandwidth(), and handle_get_status_vote().

Variable Documentation

srv_meth_pref_precompressed

compress_method_t srv_meth_pref_precompressed[]

static

Initial value:

= {
  LZMA_METHOD,
  ZSTD_METHOD,
  ZLIB_METHOD,
  GZIP_METHOD,
  NO_METHOD
}

Array of compression methods to use (if supported) for serving precompressed data, ordered from best to worst.

Definition at line 203 of file dircache.c.

Referenced by find_best_compression_method(), find_best_consensus(), and find_best_diff().

srv_meth_pref_streaming_compression

compress_method_t srv_meth_pref_streaming_compression[]

static

Initial value:

= {
  ZSTD_METHOD,
  ZLIB_METHOD,
  GZIP_METHOD,
  NO_METHOD
}

Array of compression methods to use (if supported) for serving streamed data, ordered from best to worst.

Definition at line 213 of file dircache.c.

Referenced by find_best_compression_method().

url_table

const url_table_ent_t url_table[]

static

Initial value:

= {
  { "/tor/", 0, handle_get_frontpage },
  { "/tor/status-vote/current/consensus", 1, handle_get_current_consensus },
  { "/tor/status-vote/current/", 1, handle_get_status_vote },
  { "/tor/status-vote/next/bandwidth", 0, handle_get_next_bandwidth },
  { "/tor/status-vote/next/", 1, handle_get_status_vote },
  { "/tor/micro/d/", 1, handle_get_microdesc },
  { "/tor/server/", 1, handle_get_descriptor },
  { "/tor/extra/", 1, handle_get_descriptor },
  { "/tor/keys/", 1, handle_get_keys },
  { "/tor/hs/3/", 1, handle_get_hs_descriptor_v3 },
  { "/tor/robots.txt", 0, handle_get_robots },
  { "/tor/networkstatus-bridges", 0, handle_get_networkstatus_bridges },
  { NULL, 0, NULL },
}

Table for handling GET requests.

Definition at line 368 of file dircache.c.

Referenced by directory_handle_command_get().