Dump Labeled Data (including cloud weights) at export

[OmerShubi]

/VISL2_net/addalin/efs/SHDOM/exports/2017_05_19/2017_05_19_10_30_00_no_rad_hdr_copyfordbg/

add option to export to dump labeleddata.pkl.
Includes:
Image, Mask, Sunmask, Sunshader Mask, Cloud weights.

/VISL2_net/addalin/code/pyshdom_C2/shdom/sensors.py

class RGBSensorNetwork(Sensor):
"""A network of cameras based on the 'real' camera network.

Args:
    data_dir (str): Path to reconstruction data.
    measurements (list of arrays): The measurements. These are currently used only
        for masking nan pixels.
    ignore_angles (optional, bool): Ignore the THETA/PHI pixel angles.
    ignore_sunblock (optional, bool): Ignore the (gaussian) sun mask.
    subsample (optional, int): Subsampling period.
    eleveation_weight (optional, float): Weight down the horizon.
    mask_measurement (optional, float): Mask measurements higher than this value.
    erode_mask (optional, int): Erode the mask.
    weights_color (tuple): Weight colors differently.
"""
def __init__(
        self,
        data_dir,
        indices=None,
        measurements=None,
        ignore_angles=False,
        ignore_sunblock=False,
        subsample=-1,
        elevation_weight=-1,
        mask_measurement=-1,
        erode_mask=-1,
        weights_color=(1, 1, 1),
        *args,
        **kwds):

    super(RGBSensorNetwork, self).__init__(*args, **kwds)

    with open(os.path.join(data_dir, 'export_data.pkl'), 'rb') as f:
        datas = cPickle.load(f)

    if subsample < 1:
        #
        # For some strange reason, if I don't subsample (even by 1),
        # then all shadom directions are transposed.
        #
        subsample = 1
        logging.warn("Subsampling by 1 to avoid strange bug in directions.")

    camera_ids = sorted(datas.keys())

    if indices is None:
        indices = range(len(camera_ids))

    for id_index in indices:
        cam_id = camera_ids[id_index]
        for color_channel in range(3):
            data = datas[cam_id]
            extra_data = data["extra_data"]

            if ignore_angles:
                Y_shdom, X_shdom = np.meshgrid(
                    np.linspace(-1, 1, 301),
                    np.linspace(-1, 1, 301)
                )
                PHI = math.pi + np.arctan2(Y_shdom, X_shdom)
                PSI = -math.pi + math.pi/2 * np.sqrt(X_shdom**2 + Y_shdom**2)
            else:
                PHI = data['PHI']
                PSI = data['PSI']

            #
            # Prepare the MASK (erode it if requested).
            #
            MASK = data["MASK"].astype(np.uint8)
            MASK[MASK<1] = 0
            MASK[MASK>1] = 1
            if erode_mask > 0:
                kernel = np.ones((3, 3), np.uint8)
                MASK = cv2.erode(MASK, kernel, iterations=erode_mask)
            MASK = MASK.astype(np.bool)

            SUN_MASK = data["SUN_MASK"]

            #
            # In case of forward simulation there are no measurements.
            # Use zeros to simplify the implementation of the code.
            #
            if measurements is not None:
                #
                # Mask high measurements (sun region).
                # Note:
                # The masking is done on all channels together
                # so that all channels will have the same mask.
                #
                if mask_measurement > 0:
                    values_mask = measurements[id_index] < mask_measurement
                    values_mask = values_mask.min(axis=-1)
                    MASK = MASK & values_mask

                measurement = measurements[id_index][..., color_channel]
            else:
                measurement = np.zeros_like(PHI)

            if subsample > 0:
                logging.info(
                    "Subsampling the measurements by {}".format(subsample)
                )

                #
                # Subsample the sensor smoothly.
                #
                PHI, PSI, SUN_MASK, measurement = \
                    subsampleSensor(subsample, PHI, PSI, SUN_MASK, measurement)

                #
                # The mask is sampled "not" smoothly.
                #
                MASK = MASK[::subsample, ::subsample]

            shape = PHI.shape

            self.measurements.append(measurement)
            self.shape_array.append(shape)
            self.phi_array.append(PHI)
            self.psi_array.append(PSI)
            self.x_array.append(extra_data["x"]*np.ones(shape=shape)/1000.0)
            self.y_array.append(extra_data["y"]*np.ones(shape=shape)/1000.0)
            self.z_array.append(extra_data["z"]*np.ones(shape=shape)/1000.0)

            #
            # Mask any nan values in the images.
            #
            nan_mask = ~np.isnan(measurement)
            self.mask_array.append(MASK & nan_mask)

            #
            # Scale by view angle.
            #
            if elevation_weight > 0:
                eps = 1e-8
                angle_scaling = -np.cos(PSI)
                angle_scaling[angle_scaling<eps] = eps
                angle_scaling = angle_scaling**elevation_weight
            else:
                angle_scaling = np.ones_like(PSI)

            #
            # Weight the scaling by channel.
            # This is used for putting more weights on the Blue channel in
            # hope to improve the reconstruction of the clouds.
            #
            angle_scaling = angle_scaling * weights_color[color_channel]
            if ignore_sunblock:
                self.weights_array.append(angle_scaling)
            else:
                self.weights_array.append(SUN_MASK*angle_scaling)

    #
    # SHDOM cant handle negative heights
    # move the origin to the lowest camera position
    #
    minz = np.min(map(lambda x: x.min(), self.z_array))
    self.z_array = map(lambda x: x-minz, self.z_array)

    #
    # Prepare perturbation.
    #
    if self.perturbate:
        self._prepare_camera_perturbation()

    #
    # Flatten the data arrays.
    #
    self._flatten_arrays()

/home/addalin/code/pyshdom_C2/scripts/camera_array_cloud_retrieval_RGB.py
take imports from above and sensors, on visl 135, pyshdom_new conda env

radiometric_calibration = dict(
may=np.array([0.00243353, 0.00214622, 0.00221308]),
sep=np.array([0.00394748, 0.0036557, 0.00418327]),
)

    # Load measurements
    # Note:
    # I scale the measurements by the SHDOM_COEFF. Theoretically it would
    # make sense to apply this to the solarflux input to the SHDOM algorithm.
    # This would simplify handling real measurements and simulated images
    # in the same way. Practically this is less stable SHDOM wise.
    #
    measured_images = loadRGBmeasurements(
        base_path,
        scaling=radiometric_calibration[radiometric_set]
    )

radiometric_set= 'sep'

#
# Initialize Cameras.
#
camera_network = RGBSensorNetwork(
    data_dir=base_path,
    indices=camera_ids,
    measurements=measured_images,
    ignore_sunblock=ignore_sunblock,
    subsample=subsample,
    elevation_weight=elevation_weight,
    weights_color=(0.5, 0.5, 2.) if args.weight_blue else (1, 1, 1)
) 	

EXPORT

/home/shubi/PycharmProjects/cameranetwork/CameraNetwork/export.py