Merge pull request #15 from sdimitro/delphix-6.0-master-sync

Manually sync 6.0/stage with master

configure.ac

@@ -1,6 +1,6 @@
 dnl Process this file with autoconf to produce a configure script.
 dnl
-dnl Copyright (C) 2014-2017 Petr Tesarik <[email protected]>
+dnl Copyright (C) 2014-2020 Petr Tesarik <[email protected]>
 dnl
 dnl This file is part of libkdumpfile.
 dnl
@@ -19,7 +19,7 @@ dnl  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 dnl Package release versioning
 m4_define([pkg_major_version], [0])
-m4_define([pkg_minor_version], [3])
+m4_define([pkg_minor_version], [4])
 m4_define([pkg_micro_version], [0])
 m4_define([pkg_version],
           [pkg_major_version.pkg_minor_version.pkg_micro_version])
@@ -108,8 +108,6 @@ AC_CONFIG_FILES([
 	src/addrxlat/Makefile
 	src/kdumpfile/Makefile
 	python/Makefile
-	python/addrxlat/Makefile
-	python/kdumpfile/Makefile
 	tests/Makefile
 	libaddrxlat.pc
 	libkdumpfile.pc

debian/control

@@ -17,5 +17,5 @@ Build-Depends: autoconf,
 
 Package: libkdumpfile
 Architecture: any
-Depends: ${shlibs:Depends}, ${python3:Depends}
+Depends: libsnappy-dev, ${shlibs:Depends}, ${python3:Depends}
 Description: Kernel coredump file access

include/libkdumpfile/addrxlat.h.in

@@ -592,6 +592,12 @@ typedef struct _addrxlat_paging_form {
 /** Parameters of page table translation. */
 typedef struct _addrxlat_param_pgt {
 	addrxlat_fulladdr_t root;  /**< Root page table address. */
+
+	/** Page table entry mask.
+	 * Bits to be masked off the raw value. Zero means no masking.
+	 */
+	addrxlat_pte_t pte_mask;
+
 	addrxlat_paging_form_t pf; /**< Paging form. */
 } addrxlat_param_pgt_t;
 

include/libkdumpfile/kdumpfile.h.in

@@ -71,6 +71,17 @@ extern "C" {
  */
 typedef uint_fast64_t kdump_num_t;
 
+/** @name fprintf() macros for the libkdumpfile number type
+ * @{
+ *
+ * POSIX @c PRI_xxx style macros for use with @ref kdump_num_t.
+ */
+#define KDUMP_PRIoNUM	PRIoFAST64 /**< Octal @c kdump_num_t */
+#define KDUMP_PRIuNUM	PRIuFAST64 /**< Decimal @c kdump_num_t */
+#define KDUMP_PRIxNUM	PRIxFAST64 /**< Lowercase hex @c kdump_num_t */
+#define KDUMP_PRIXNUM	PRIXFAST64 /**< Uppercase hex @c kdump_num_t */
+/* @} */
+
 /**  Type of a physical or virtual address.
  *
  * This type is large enough to hold any possible address type on any
@@ -97,6 +108,18 @@ typedef kdump_addr_t kdump_paddr_t;
  */
 typedef kdump_addr_t kdump_vaddr_t;
 
+/** @name fprintf() macros for the libkdumpfile address type
+ * @{
+ *
+ * POSIX @c PRI_xxx style macros for use with @ref kdump_addr_t,
+ * @ref kdump_paddr_t and @ref kdump_vaddr_t.
+ */
+#define KDUMP_PRIoADDR	ADDRXLAT_PRIoADDR /**< Octal address */
+#define KDUMP_PRIuADDR	ADDRXLAT_PRIuADDR /**< Decimal address */
+#define KDUMP_PRIxADDR	ADDRXLAT_PRIxADDR /**< Lowercase hex address */
+#define KDUMP_PRIXADDR	ADDRXLAT_PRIXADDR /**< Uppercase hex address */
+/* @} */
+
 /**  Representation of a dump file.
  *
  * The context contains all information needed to work with a dump file.
@@ -138,6 +161,24 @@ typedef enum _kdump_byte_order {
 	KDUMP_LITTLE_ENDIAN = ADDRXLAT_LITTLE_ENDIAN,
 } kdump_byte_order_t;
 
+/**  File mmap policy.
+ *
+ * Control if and how mmap(2) should be used to access data from the
+ * underlying core file.
+ *
+ * @sa KDUMP_ATTR_FILE_MMAP_POLICY
+ */
+typedef enum _kdump_mmap_policy {
+	KDUMP_MMAP_NEVER,	/**< Never use mmap(2), always use read(2). */
+	KDUMP_MMAP_ALWAYS,	/**< Always use mmap(2), never use read(2). */
+	KDUMP_MMAP_TRY,		/**< Try mmap(2), fall back to read(2). */
+
+	/** Same as @c KDUMP_MMAP_TRY, but change to @c KDUMP_MMAP_NEVER
+	 *  or @c KDUMP_MMAP_ALWAYS based on the result of the next read.
+	 */
+	KDUMP_MMAP_TRY_ONCE,
+} kdump_mmap_policy_t;
+
 /**  Type of a Xen dump.
  * @sa KDUMP_ATTR_XEN_TYPE
  */
@@ -406,6 +447,35 @@ kdump_status kdump_bmp_find_clear(
  */
 typedef struct _kdump_blob kdump_blob_t;
 
+/** Create a new blob object.
+ * @param data  Raw data buffer (can be @c NULL).
+ * @param size  Length of data at @c data.
+ * @returns     New blob object, or @c NULL on allocation error.
+ *
+ * Create a new blob object from a dynamically allocated memory buffer.
+ * Keep in mind that the buffer will be eventually de-allocated with
+ * free(3) when the blob reference count drops to zero.
+ *
+ * If @c data is @c NULL, then the blob does not contain any data.
+ *
+ * The new object's reference count is initialized to 1 (the caller
+ * receives one reference), and the memory buffer is not pinned.
+ */
+kdump_blob_t *kdump_blob_new(void *data, size_t size);
+
+/** Create a new blob object from a copy of a buffer.
+ * @param data  Raw data buffer (can be @c NULL).
+ * @param size  Length of data at @c data.
+ * @returns     New blob object, or @c NULL on allocation error.
+ *
+ * Create a new blob object by duplicating data from a buffer. The
+ * data is copied into a newly allocated buffer which is owned by
+ * the blob object itself.
+ *
+ * @sa kdump_blob_new
+ */
+kdump_blob_t *kdump_blob_new_dup(const void *data, size_t size);
+
 /** Increment kdump blob reference counter.
  * @param blob   Binary object.
  * @returns      New reference count.
@@ -923,24 +993,36 @@ void kdump_attr_iter_end(kdump_ctx_t *ctx, kdump_attr_iter_t *iter);
 /** Fill excluded pages with zeroes? */
 #define KDUMP_ATTR_ZERO_EXCLUDED "file.zero_excluded"
 
+/** Policy for using mmap vs. read.
+ * Default is @c KDUMP_MMAP_TRY.
+ * @sa kdump_mmap_policy_t
+ */
+#define KDUMP_ATTR_FILE_MMAP_POLICY	"file.mmap_policy"
+
 /**  Get VMCOREINFO raw data.
  * @param ctx  Dump file object.
- * @param raw  Filled with raw VMCOREINFO string on success.
+ * @param raw  Filled with a copy of the raw VMCOREINFO string on success.
  * @returns    Error status.
  *
  * The output string is always NUL-terminated, but if there was a NUL
  * byte inside VMCOREINFO, there is no way to know the full length.
+ *
+ * The output string is dynamically allocated and must be eventually
+ * deallocated by the caller using @c free(3).
  */
-kdump_status kdump_vmcoreinfo_raw(kdump_ctx_t *ctx, const char **raw);
+kdump_status kdump_vmcoreinfo_raw(kdump_ctx_t *ctx, char **raw);
 
 /**  Get a VMCOREINFO line.
  * @param ctx       Dump file object.
  * @param[in] key   Name of the VMCOREINFO variable (part before '=').
  * @param[out] val  Filled with the part after (first) '='.
  * @returns         Error status.
+ *
+ * The output string is dynamically allocated and must be eventually
+ * deallocated by the caller using @c free(3).
  */
 kdump_status kdump_vmcoreinfo_line(kdump_ctx_t *ctx, const char *key,
-				   const char **val);
+				   char **val);
 
 /**  Get VMCOREINFO symbol value.
  * @param ctx            Dump file object.

python/.gitignore

@@ -1,3 +1,9 @@
+# generated by Makefile
+setup.cfg
+
+# distutils build
+build
+
 # Test results
 *.log
 *.trs

python/Makefile.am

@@ -23,17 +23,42 @@ AM_CPPFLAGS = -I$(top_builddir)/include
 AM_CFLAGS = $(PYTHON_CFLAGS)
 AM_LDFLAGS = -module -avoid-version
 
-pyexec_LTLIBRARIES = _addrxlat.la _kdumpfile.la
+installed_list = installed.list
 
-_addrxlat_la_SOURCES = addrxlat.c
-_addrxlat_la_LDFLAGS = $(AM_LDFLAGS) -export-symbols $(srcdir)/addrxlat.sym
-_addrxlat_la_LIBADD = $(PYTHON_LIBS) $(top_builddir)/src/addrxlat/libaddrxlat.la
-EXTRA__addrxlat_la_DEPENDENCIES = addrxlat.sym
+setup.cfg: Makefile
+	$(AM_V_GEN)
+	$(AM_V_at)echo "[kdumpfile]" > $@
+	$(AM_V_at)echo "version=$(VERSION)" >> $@
+	$(AM_V_at)echo "srcdir=$(srcdir)" >> $@
+	$(AM_V_at)echo "top_builddir=$(top_builddir)" >> $@
+	$(AM_V_at)echo >> $@
+	$(AM_V_at)echo "[build_ext]" >> $@
+	$(AM_V_at)echo "libtool=$(LIBTOOL)" >> $@
+	$(AM_V_at)echo "pyexecdir=$(pyexecdir)" >> $@
+	$(AM_V_at)echo >> $@
+	$(AM_V_at)echo "[install_lib]" >> $@
+	$(AM_V_at)echo "libtool_install=$(LIBTOOL) --mode=install $(INSTALL)" >> $@
 
-_kdumpfile_la_SOURCES = kdumpfile.c
-_kdumpfile_la_LDFLAGS = $(AM_LDFLAGS) -export-symbols $(srcdir)/kdumpfile.sym
-_kdumpfile_la_LIBADD = $(PYTHON_LIBS) $(top_builddir)/src/kdumpfile/libkdumpfile.la
-EXTRA__kdumpfile_la_DEPENDENCIES = kdumpfile.sym
+all-local: setup.cfg
+	$(PYTHON) $(srcdir)/setup.py build
+
+install-exec-local: setup.cfg
+	$(PYTHON) $(srcdir)/setup.py install \
+		--root "$(DESTDIR)"/ \
+		--install-purelib $(pythondir) \
+		--install-platlib $(pyexecdir) \
+		--record $(installed_list)
+
+uninstall-local: $(installed_list)
+	while read f ; do \
+	    $(LIBTOOL) --mode=uninstall rm -f "$(DESTDIR)/$$f" ; \
+	done < $<
+
+mostlyclean-local: setup.cfg
+	$(PYTHON) $(srcdir)/setup.py clean --all
+	rm $<
+
+CLEANFILES = $(installed_list)
 
 check_LTLIBRARIES = _test_addrxlat.la
 
@@ -45,13 +70,22 @@ _test_addrxlat_la_LDFLAGS = \
 _test_addrxlat_la_LIBADD = $(PYTHON_LIBS) $(top_builddir)/src/addrxlat/libaddrxlat.la
 EXTRA__test_addrxlat_la_DEPENDENCIES = test_addrxlat.sym
 
-dist_noinst_DATA = \
-	addrxlat.sym \
-	kdumpfile.sym \
-	test_addrxlat.sym \
+dist_noinst_SCRIPTS = \
+	setup.py \
+	libtoolize.py \
+	addrxlat/__init__.py \
+	addrxlat/exceptions.py \
+	kdumpfile/__init__.py \
+	kdumpfile/exceptions.py \
+	kdumpfile/views.py \
 	showxlat.py \
 	vtop.py
 
+dist_noinst_DATA = \
+	addrxlat.c \
+	kdumpfile.c \
+	test_addrxlat.sym
+
 noinst_HEADERS = \
 	addrxlatmod.h
 
@@ -68,5 +102,3 @@ LOG_COMPILER = $(PYTHON)
 AM_TESTS_ENVIRONMENT = \
 eval " $$($(LIBTOOL) --config)"; \
 PYTHONPATH="$$srcdir:$$objdir"
-
-SUBDIRS = addrxlat kdumpfile

python/addrxlat.c

@@ -521,6 +521,39 @@ set_addrspace(PyObject *self, PyObject *value, void *data)
 	return 0;
 }
 
+/** Getter for the addrxlat_pte_t type.
+ * @param self  any object
+ * @param data  offset of the addrxlat_pte_t member
+ * @returns     PyLong object (or @c NULL on failure)
+ */
+static PyObject *
+get_pte(PyObject *self, void *data)
+{
+	Py_ssize_t off = (intptr_t)data;
+	addrxlat_pte_t *pval = (addrxlat_pte_t*)((char*)self + off);
+	return PyLong_FromUnsignedLongLong(*pval);
+}
+
+/** Setter for the addrxlat_pte_t type.
+ * @param self   any object
+ * @param value  new value (a @c PyLong or @c PyInt)
+ * @param data   offset of the addrxlat_pte_t member
+ * @returns      zero on success, -1 otherwise
+ */
+static int
+set_pte(PyObject *self, PyObject *value, void *data)
+{
+	Py_ssize_t off = (intptr_t)data;
+	addrxlat_pte_t *pval = (addrxlat_pte_t*)((char*)self + off);
+	unsigned long long pte = Number_AsUnsignedLongLongMask(value);
+
+	if (PyErr_Occurred())
+		return -1;
+
+	*pval = pte;
+	return 0;
+}
+
 /** An object with a C pointer.
  * This object is used to create a Python object from a C pointer to
  * the corresponding libaddrxlat object passed as a _C_POINTER argument.
@@ -3110,10 +3143,15 @@ static fulladdr_loc pgtmeth_root_loc = {
 	"root"
 };
 
+PyDoc_STRVAR(pgtmeth_pte_mask__doc__,
+"page table entry mask");
+
 static PyGetSetDef pgtmeth_getset[] = {
 	{ "kind", meth_get_kind, 0, pgtmeth_kind__doc__ },
 	{ "root", get_fulladdr, set_fulladdr, pgtmeth_root__doc__,
 	  &pgtmeth_root_loc },
+	{ "pte_mask", get_pte, set_pte, pgtmeth_pte_mask__doc__,
+	  OFFSETOF_PTR(meth_object, meth.param.pgt.pte_mask) },
 	{ "pte_format", pgtmeth_get_pte_format, pgtmeth_set_pte_format,
 	  pgtmeth_pte_format__doc__ },
 	{ "fields", pgtmeth_get_fields, pgtmeth_set_fields,