Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DB corruption fixes #917

Open
wants to merge 10 commits into
base: master
Choose a base branch
from
Open

DB corruption fixes #917

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
672aea0
Fixes to avoid null calls to DB
hamza-vd May 10, 2023
3be6661
Enable OpenSRPDatabaseErrorHandler in Repository class
hamza-vd May 11, 2023
4feaf49
Make Repository constructor backward compatible
hamza-vd May 11, 2023
1346929
Refactor user settings tasks to avoid LockingBackgroundTask
hamza-vd May 12, 2023
0ebf753
Compare server and device locations count before fetching all locations
qiarie May 15, 2023
3e39f05
Refactor all locations fetch to use last serverVersion
qiarie May 15, 2023
4a1fe61
Add tests
hamza-vd May 17, 2023
47744b4
Fix compilation errors in tests
hamza-vd May 17, 2023
cda4e3c
Fix failing tests
hamza-vd May 18, 2023
41cc6d5
Increase Code Coverage
hamza-vd May 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions opensrp-core/src/main/java/org/smartregister/SyncConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,4 +236,12 @@ public boolean validateUserAssignments() {
public boolean skipUnsyncedTasksOnFetchFromServer(){
return false;
}

/*
* Override this method to make the app aware
* is P2P sync is enabled or disabled for it
*/
public boolean isP2PEnabled() {
return true;
}
}
123 changes: 75 additions & 48 deletions opensrp-core/src/main/java/org/smartregister/repository/EventClientRepository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@



package org.smartregister.repository;

import static org.smartregister.AllConstants.ROWID;
Expand Down Expand Up @@ -193,7 +190,7 @@ public static void dropIndexes(SQLiteDatabase db, BaseTable table) {
db.execSQL("DROP INDEX " + cursor.getString(0));
}
} catch (Exception e) {
Timber.e(EventClientRepository.class.getName(), "SQLException", e);
Timber.e(e, "SQLException");
} finally {
if (cursor != null)
cursor.close();
Expand Down Expand Up @@ -1227,7 +1224,11 @@ public List<String> getUnValidatedEventFormSubmissionIds(int limit) {
if (cursor != null && cursor.getCount() > 0 && cursor.moveToFirst()) {
while (!cursor.isAfterLast()) {
String id = cursor.getString(0);
ids.add(id);

if (StringUtils.isNotBlank(id))
ids.add(id);
else
Timber.e("Unvalidated Event FormSubmissionId is empty/NULL");

cursor.moveToNext();
}
Expand Down Expand Up @@ -1267,7 +1268,11 @@ public List<String> getUnValidatedClientBaseEntityIds(int limit) {
if (cursor != null && cursor.getCount() > 0 && cursor.moveToFirst()) {
while (!cursor.isAfterLast()) {
String id = cursor.getString(0);
ids.add(id);

if (StringUtils.isNotBlank(id))
ids.add(id);
else
Timber.e("Unvalidated Client BaseEntityId is empty/NULL");

cursor.moveToNext();
}
Expand Down Expand Up @@ -1312,7 +1317,8 @@ public void markAllAsUnSynced() {
ContentValues values = new ContentValues();
values.put(client_column.baseEntityId.name(), beid);
values.put(client_column.syncStatus.name(), BaseRepository.TYPE_Unsynced);
values.put(ROWID, maxRowId++);
if (CoreLibrary.getInstance().context().hasForeignEvents())
values.put(ROWID, maxRowId++);

getWritableDatabase().update(clientTable.name(),
values,
Expand All @@ -1336,7 +1342,8 @@ public void markAllAsUnSynced() {
ContentValues values = new ContentValues();
values.put(event_column.baseEntityId.name(), beid);
values.put(event_column.syncStatus.name(), BaseRepository.TYPE_Unsynced);
values.put(ROWID, maxRowId++);
if (CoreLibrary.getInstance().context().hasForeignEvents())
values.put(ROWID, maxRowId++);

getWritableDatabase().update(eventTable.name(),
values,
Expand Down Expand Up @@ -1970,7 +1977,8 @@ public void addorUpdateClient(String baseEntityId, JSONObject jsonObject, String
getClientRelationShip(baseEntityId, jsonObject);
long affected;
if (checkIfExists(clientTable, baseEntityId)) {
values.put(ROWID, getMaxRowId(clientTable) + 1);
if (Utils.isP2PEnabled())
values.put(ROWID, getMaxRowId(clientTable) + 1);

affected = getWritableDatabase().update(clientTable.name(),
values,
Expand Down Expand Up @@ -2024,8 +2032,8 @@ public void addEvent(String baseEntityId, JSONObject jsonObject, String syncStat
jsonObject.getString(event_column
.formSubmissionId
.name()))) {

values.put(ROWID, getMaxRowId(eventTable) + 1);
if (Utils.isP2PEnabled())
values.put(ROWID, getMaxRowId(eventTable) + 1);
affected = getWritableDatabase().update(eventTable.name(),
values,
event_column.formSubmissionId.name() + "=?",
Expand All @@ -2040,7 +2048,7 @@ public void addEvent(String baseEntityId, JSONObject jsonObject, String syncStat

}
} else {
// a case here would be if an event comes from openmrs
// a case here would be if an event comes from openmrs
affected = getWritableDatabase().insert(eventTable.name(), null, values);
}

Expand All @@ -2063,7 +2071,8 @@ public void markEventAsProcessed(String formSubmissionId) {

ContentValues values = new ContentValues();
values.put(event_column.syncStatus.name(), BaseRepository.TYPE_Unsynced);
values.put(ROWID, getMaxRowId(eventTable) + 1);
if (Utils.isP2PEnabled())
values.put(ROWID, getMaxRowId(eventTable) + 1);

getWritableDatabase().update(eventTable.name(),
values,
Expand All @@ -2080,7 +2089,8 @@ public void markEventAsSynced(String formSubmissionId) {

ContentValues values = new ContentValues();
values.put(event_column.syncStatus.name(), BaseRepository.TYPE_Synced);
values.put(ROWID, getMaxRowId(eventTable) + 1);
if (Utils.isP2PEnabled())
values.put(ROWID, getMaxRowId(eventTable) + 1);

getWritableDatabase().update(eventTable.name(),
values,
Expand All @@ -2098,7 +2108,8 @@ public void markClientAsSynced(String baseEntityId) {
ContentValues values = new ContentValues();
values.put(client_column.baseEntityId.name(), baseEntityId);
values.put(client_column.syncStatus.name(), BaseRepository.TYPE_Synced);
values.put(ROWID, getMaxRowId(clientTable) + 1);
if (Utils.isP2PEnabled())
values.put(ROWID, getMaxRowId(clientTable) + 1);

getWritableDatabase().update(clientTable.name(),
values,
Expand All @@ -2112,56 +2123,72 @@ public void markClientAsSynced(String baseEntityId) {

public void markEventValidationStatus(String formSubmissionId, boolean valid) {
try {
ContentValues values = new ContentValues();
values.put(event_column.formSubmissionId.name(), formSubmissionId);
values.put(event_column.validationStatus.name(), valid ? TYPE_Valid : TYPE_InValid);
if (!valid) {
values.put(event_column.syncStatus.name(), TYPE_Unsynced);
}
values.put(ROWID, getMaxRowId(eventTable) + 1);

getWritableDatabase().update(eventTable.name(),
values,
event_column.formSubmissionId.name() + " = ?",
new String[]{formSubmissionId});
if (StringUtils.isNotBlank(formSubmissionId)) {
ContentValues values = new ContentValues();
values.put(event_column.formSubmissionId.name(), formSubmissionId);
values.put(event_column.validationStatus.name(), valid ? TYPE_Valid : TYPE_InValid);
if (!valid) {
values.put(event_column.syncStatus.name(), TYPE_Unsynced);
}
if (Utils.isP2PEnabled())
values.put(ROWID, getMaxRowId(eventTable) + 1);

getWritableDatabase().update(
eventTable.name(),
values,
event_column.formSubmissionId.name() + " = ?",
new String[]{formSubmissionId}
);
} else {
Timber.e("markEventValidationStatus: Event formSubmissionId is empty/NULL");
}
} catch (Exception e) {
Timber.e(e);
}
}


public void markClientValidationStatus(String baseEntityId, boolean valid) {
try {
ContentValues values = new ContentValues();
values.put(client_column.baseEntityId.name(), baseEntityId);
values.put(client_column.validationStatus.name(), valid ? TYPE_Valid : TYPE_InValid);
if (!valid) {
values.put(client_column.syncStatus.name(), TYPE_Unsynced);
}

values.put(ROWID, getMaxRowId(clientTable) + 1);

getWritableDatabase().update(clientTable.name(),
values,
client_column.baseEntityId.name() + " = ?",
new String[]{baseEntityId});
if (StringUtils.isNotBlank(baseEntityId)) {
ContentValues values = new ContentValues();
values.put(client_column.baseEntityId.name(), baseEntityId);
values.put(client_column.validationStatus.name(), valid ? TYPE_Valid : TYPE_InValid);
if (!valid) {
values.put(client_column.syncStatus.name(), TYPE_Unsynced);
}
if (Utils.isP2PEnabled())
values.put(ROWID, getMaxRowId(clientTable) + 1);

getWritableDatabase().update(
clientTable.name(),
values,
client_column.baseEntityId.name() + " = ?",
new String[]{baseEntityId}
);
} else {
Timber.e("markClientValidationStatus: Client baseEntityId is empty/NULL");
}
} catch (Exception e) {
Timber.e(e);
}
}

public void markEventAsTaskUnprocessed(String formSubmissionId) {
try {
ContentValues values = new ContentValues();
values.put(client_column.syncStatus.name(), TYPE_Task_Unprocessed);
values.put(ROWID, getMaxRowId(eventTable) + 1);
if (StringUtils.isNotBlank(formSubmissionId)) {
ContentValues values = new ContentValues();
values.put(client_column.syncStatus.name(), TYPE_Task_Unprocessed);

getWritableDatabase().update(eventTable.name(),
values,
event_column.formSubmissionId.name() + " = ?",
new String[]{formSubmissionId});
if (Utils.isP2PEnabled())
values.put(ROWID, getMaxRowId(eventTable) + 1);

getWritableDatabase().update(eventTable.name(),
values,
event_column.formSubmissionId.name() + " = ?",
new String[]{formSubmissionId});
} else {
Timber.e("markEventAsTaskUnprocessed: Event formSubmissionId is empty/NULL");
}
} catch (Exception e) {
Timber.e(e);
}
Expand Down
4 changes: 1 addition & 3 deletions opensrp-core/src/main/java/org/smartregister/repository/LocationRepository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package org.smartregister.repository;

import android.content.ContentValues;

import androidx.annotation.NonNull;

import com.google.gson.Gson;
Expand All @@ -13,8 +14,6 @@
import org.smartregister.domain.Location;
import org.smartregister.domain.LocationProperty;
import org.smartregister.domain.LocationTag;
import org.smartregister.domain.PhysicalLocation;
import org.smartregister.pathevaluator.dao.LocationDao;
import org.smartregister.util.PropertiesConverter;

import java.util.ArrayList;
Expand Down Expand Up @@ -301,5 +300,4 @@ public List<Location> getLocationsByTagName(String tagName) {

return getLocationsByIds(locationIds);
}

}
26 changes: 22 additions & 4 deletions opensrp-core/src/main/java/org/smartregister/repository/Repository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

import android.content.Context;
import androidx.annotation.VisibleForTesting;

import net.sqlcipher.DatabaseErrorHandler;
import net.sqlcipher.database.SQLiteDatabase;
import net.sqlcipher.database.SQLiteDatabaseHook;
import net.sqlcipher.database.SQLiteException;
Expand Down Expand Up @@ -55,8 +57,13 @@ public void postKey(SQLiteDatabase database) {
};

public Repository(Context context, Session session, DrishtiRepository... repositories) {
this(context, session, new OpenSRPDatabaseErrorHandler(), repositories);
}

public Repository(Context context, Session session, DatabaseErrorHandler databaseErrorHandler,
DrishtiRepository... repositories) {
super(context, (session != null ? session.repositoryName() : AllConstants.DATABASE_NAME),
null, 1, hook);
null, 1, hook, databaseErrorHandler);
this.repositories = repositories;
this.context = context;
this.session = session;
Expand All @@ -71,15 +78,26 @@ public Repository(Context context, Session session, DrishtiRepository... reposit
}
}

public Repository(Context context, Session session, CommonFtsObject commonFtsObject,
public Repository(Context context, Session session, CommonFtsObject commonFtsObject, DrishtiRepository... repositories) {
this(context, session, commonFtsObject, new OpenSRPDatabaseErrorHandler(), repositories);
}


public Repository(Context context, Session session, CommonFtsObject commonFtsObject, DatabaseErrorHandler databaseErrorHandler,
DrishtiRepository... repositories) {
this(context, session, repositories);
this(context, session, databaseErrorHandler, repositories);
this.commonFtsObject = commonFtsObject;
}

public Repository(Context context, String dbName, int version, Session session,
CommonFtsObject commonFtsObject, DrishtiRepository... repositories) {
super(context, dbName, null, version, hook);
this(context, dbName, version, session, commonFtsObject, new OpenSRPDatabaseErrorHandler(), repositories);
}

public Repository(Context context, String dbName, int version, Session session,
CommonFtsObject commonFtsObject, DatabaseErrorHandler databaseErrorHandler,
DrishtiRepository... repositories) {
super(context, dbName, null, version, hook, databaseErrorHandler);
this.dbName = dbName;
this.repositories = repositories;
this.context = context;
Expand Down
34 changes: 25 additions & 9 deletions ...p-core/src/main/java/org/smartregister/repository/helper/OpenSRPDatabaseErrorHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,35 +1,51 @@
package org.smartregister.repository.helper;

import android.util.Log;

import net.sqlcipher.DatabaseErrorHandler;
import net.sqlcipher.database.SQLiteDatabase;

import java.io.File;

import timber.log.Timber;

/**
* Created by ndegwamartin on 18/07/2020.
*/
public class OpenSRPDatabaseErrorHandler implements DatabaseErrorHandler {

private final String TAG = getClass().getSimpleName();

/**
* defines the default method to be invoked when database corruption is detected.
*
* @param dbObj the {@link SQLiteDatabase} object representing the database on which corruption
* is detected.
* is detected.
*/
public void onCorruption(SQLiteDatabase dbObj) {
Log.e(TAG, "Corruption reported by sqlite on database, db file path: " + dbObj.getPath());
Timber.e("Corruption reported by sqlite on database, deleting: %s", dbObj.getPath());

if (dbObj.isOpen()) {
Log.e(TAG, "Database object for corrupted database is already open, closing");
Timber.e( "Database object for corrupted database is already open, closing");

try {
dbObj.close();
} catch (Exception e) {
Log.e(TAG, "Exception closing Database object for corrupted database, ignored", e);
/* ignored */
Timber.e(e, "Exception closing Database object for corrupted database, ignored");
}
}

deleteDatabaseFile(dbObj.getPath());
}

private void deleteDatabaseFile(String fileName) {
if (fileName == null || fileName.equalsIgnoreCase(":memory:") || fileName.trim().length() == 0) {
Timber.e("Cannot delete database. Provided filename is not valid: %s", fileName);
return;
}
Timber.e( "deleting the database file: %s", fileName);
try {
//noinspection ResultOfMethodCallIgnored
new File(fileName).delete();
} catch (Exception e) {
/* print warning and ignore exception */
Timber.w(e, "delete failed");
}
}
}
Loading