Como construir um modelo de detecção de fraudes em Java usando o aprendizado profundo4J

1
<dependencies>  
2
    <dependency>  
3
        <groupId>org.deeplearning4j</groupId>  
4
        <artifactId>deeplearning4j-core</artifactId>  
5
        <version>1.0.0-M2.1</version>  
6
    </dependency>  
7
    <dependency>  
8
        <groupId>org.nd4j</groupId>  
9
        <artifactId>nd4j-native-platform</artifactId>  
10
        <version>1.0.0-M2.1</version>  
11
    </dependency>  
12
    <dependency>  
13
        <groupId>org.mongodb</groupId>  
14
        <artifactId>mongodb-driver-sync</artifactId>  
15
        <version>5.2.0</version>  
16
    </dependency>  
17
    <dependency>  
18
        <groupId>org.slf4j</groupId>  
19
        <artifactId>slf4j-simple</artifactId>  
20
        <version>2.0.16</version>  
21
    </dependency>  
22
  
23
</dependencies>

1
package com.mongodb;
2
3
import com.mongodb.client.MongoClient;
4
import com.mongodb.client.MongoClients;
5
import com.mongodb.client.MongoCollection;
6
import com.mongodb.client.MongoDatabase;
7
import org.bson.Document;
8
9
import java.util.concurrent.TimeUnit;
10
11
public class MongoDBConnector {
12
    private static final String URI = "YOUR-CONNECTION-STRING";
13
    private static final String DATABASE_NAME = "fraudDetection";
14
    private static final String COLLECTION_NAME = "transactions";
15
16
    private final MongoClient mongoClient;
17
18
    public MongoDBConnector() {
19
        MongoClientSettings settings = MongoClientSettings.builder()
20
                .applyConnectionString(new ConnectionString(URI))
21
                .applyToSocketSettings(builder ->
22
                        builder.connectTimeout(30, TimeUnit.SECONDS)
23
                                .readTimeout(30, TimeUnit.SECONDS))
24
                .build();
25
26
        mongoClient = MongoClients.create(settings);
27
    }
28
29
    public MongoCollection<Document> getCollection() {
30
        return mongoClient.getDatabase(DATABASE_NAME).getCollection(COLLECTION_NAME);
31
    }
32
33
}

1
package com.mongodb;  
2
  
3
import org.bson.Document;  
4
  
5
public class Transaction {  
6
    private double amount;  
7
    private boolean isFraudulent;  
8
  
9
    public Transaction(double amount, boolean isFraudulent) {  
10
        this.amount = amount;  
11
        this.isFraudulent = isFraudulent;  
12
    }  
13
  
14
    public double getAmount() { return amount; }  
15
    public boolean isFraudulent() { return isFraudulent; }  
16
  
17
    public Document toDocument() {  
18
        Document doc = new Document();  
19
        doc.append("amount", amount);  
20
        doc.append("isFraudulent", isFraudulent);  
21
        return doc;  
22
    }  
23
}

1
package com.mongodb;
2
3
import com.mongodb.client.MongoCollection;
4
import com.mongodb.client.model.BulkWriteOptions;
5
import com.mongodb.client.model.WriteModel;
6
import org.bson.Document;
7
8
import java.util.ArrayList;
9
import java.util.List;
10
11
public class TransactionRepository {
12
    private final MongoCollection<Document> collection;
13
14
    public TransactionRepository(MongoDBConnector connector) {
15
        this.collection = connector.getCollection();
16
    }
17
18
    public void bulkSaveTransactions(List<WriteModel<Document>> transactions) {
19
        if (!transactions.isEmpty()) {
20
            try {
21
                BulkWriteOptions options = new BulkWriteOptions().ordered(true);
22
                collection.bulkWrite(transactions, options); // Perform bulk write operation
23
            } catch (Exception e) {
24
                e.printStackTrace();
25
            }
26
        }
27
    }
28
29
    public List<Transaction> getAllTransactions() {
30
        List<Transaction> transactions = new ArrayList<>();
31
        for (Document doc : collection.find()) {
32
            double amount = doc.getDouble("amount");
33
            boolean isFraudulent = doc.getBoolean("isFraudulent");
34
            transactions.add(new Transaction(amount, isFraudulent));
35
        }
36
        return transactions;
37
    }
38
39
}

1
package com.mongodb;
2
3
import com.mongodb.client.model.WriteModel;
4
import com.mongodb.client.model.InsertOneModel;
5
import org.bson.Document;
6
7
import java.io.BufferedReader;
8
import java.io.FileReader;
9
import java.io.IOException;
10
import java.util.ArrayList;
11
import java.util.List;
12
import java.util.stream.Collectors;
13
14
public class DataPreprocessor {
15
16
    TransactionRepository transactionRepository;
17
18
    private static final int BATCH_SIZE = 500; // Define batch size for bulk writes
19
    private static final int DOCUMENT_LIMIT = 250000;  // Maximum documents to insert
20
21
    private int documentCount = 0;  // Counter for total inserted documents
22
23
    public DataPreprocessor(TransactionRepository transactionRepository) {
24
        this.transactionRepository = transactionRepository;
25
    }
26
27
    public void loadData(String filePath) throws IOException {
28
        try (BufferedReader reader = new BufferedReader(new FileReader(filePath))) {
29
            // Skip the header by reading the first line
30
            reader.readLine();
31
32
            List<String> batch = new ArrayList<>();
33
            String line;
34
35
            // Read the file line-by-line
36
            while ((line = reader.readLine()) != null) {
37
                if (documentCount >= DOCUMENT_LIMIT) {
38
                    System.out.println("Reached the document limit of " + DOCUMENT_LIMIT + ". Stopping data load.");
39
                    break;  // Stop processing when the limit is reached
40
                }
41
42
                batch.add(line);
43
                documentCount++;
44
45
                // When batch size is reached, process it
46
                if (batch.size() == BATCH_SIZE) {
47
                    processBatch(batch);
48
                    batch.clear(); // Clear the batch for the next set of lines
49
                }
50
            }
51
52
            // Process any remaining lines
53
            if (!batch.isEmpty()) {
54
                processBatch(batch);
55
            }
56
        }
57
    }
58
59
    private void processBatch(List<String> batch) {
60
        List<WriteModel<Document>> bulkOperations = batch.stream()
61
                .map(line -> {
62
                    String[] fields = line.split(",");
63
                    double amount = Double.parseDouble(fields[29]); // Adjust index as needed
64
                    boolean isFraudulent = "1".equals(fields[30]);
65
                    Transaction transaction = new Transaction(amount, isFraudulent);
66
                    return new InsertOneModel<>(transaction.toDocument());
67
                })
68
                .collect(Collectors.toList());
69
        transactionRepository.bulkSaveTransactions(bulkOperations);
70
    }
71
72
}

1
package com.mongodb;  
2
  
3
import org.deeplearning4j.nn.conf.MultiLayerConfiguration;  
4
import org.deeplearning4j.nn.conf.NeuralNetConfiguration;  
5
import org.deeplearning4j.nn.conf.layers.DenseLayer;  
6
import org.deeplearning4j.nn.conf.layers.OutputLayer;  
7
import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;  
8
import org.deeplearning4j.nn.weights.WeightInit;  
9
import org.deeplearning4j.optimize.listeners.ScoreIterationListener;  
10
import org.nd4j.evaluation.classification.Evaluation;  
11
import org.nd4j.linalg.activations.Activation;  
12
import org.nd4j.linalg.api.ndarray.INDArray;  
13
import org.nd4j.linalg.dataset.DataSet;  
14
import org.nd4j.linalg.factory.Nd4j;  
15
import org.nd4j.linalg.learning.config.Adam;  
16
import org.nd4j.linalg.lossfunctions.LossFunctions;  
17
  
18
import java.util.Collections;  
19
import java.util.List;  
20
  
21
public class FraudDetectionModel {  
22
    private MultiLayerNetwork model;  
23
24
    private static final int NUM_INPUT_FEATURES = 1; 
25
    private static final int NUM_CLASSES = 2;  
26
  
27
    public FraudDetectionModel() {  
28
        initializeModel();  
29
    }  
30
  
31
    private void initializeModel() {  
32
        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()  
33
                .seed(123)  
34
                .weightInit(WeightInit.XAVIER)  
35
                .updater(new Adam(0.001))  
36
                .list()  
37
                .layer(0, new DenseLayer.Builder()  
38
                        .nIn(NUM_INPUT_FEATURES)  
39
                        .nOut(10)  
40
                        .activation(Activation.RELU)  
41
                        .build())  
42
                .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)  
43
                        .nIn(10)  
44
                        .nOut(NUM_CLASSES)  
45
                        .activation(Activation.SOFTMAX)  
46
                        .build())  
47
                .build();  
48
  
49
        model = new MultiLayerNetwork(conf);  
50
        model.init();  
51
        model.setListeners(new ScoreIterationListener(10));  
52
    }  
53
  
54
}

1
private MultiLayerNetwork model;
2
private static final int NUM_INPUT_FEATURES = 1;
3
private static final int NUM_CLASSES = 2;

1
private void initializeModel() {
2
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
3
            .seed(123)
4
            .weightInit(WeightInit.XAVIER)
5
            .updater(new Adam(0.001))
6
            .list()

1
.layer(0, new DenseLayer.Builder()
2
        .nIn(NUM_INPUT_FEATURES)
3
        .nOut(10)
4
        .activation(Activation.RELU)
5
        .build())

1
.layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
2
        .nIn(10)
3
        .nOut(NUM_CLASSES)
4
        .activation(Activation.SOFTMAX)
5
        .build())

1
model = new MultiLayerNetwork(conf);
2
model.init();
3
model.setListeners(new ScoreIterationListener(10));

1
    private DataSet prepareTrainingData(List<Transaction> transactions) {  
2
        int numTransactions = transactions.size();  
3
        INDArray features = Nd4j.create(numTransactions, NUM_INPUT_FEATURES);  
4
        INDArray labels = Nd4j.create(numTransactions, NUM_CLASSES);  
5
        DataSet dataSet;  
6
  
7
        for (int i = 0; i < numTransactions; i++) {  
8
            Transaction transaction = transactions.get(i);  
9
  
10
            // Use transaction amount as feature  
11
            features.putScalar(new int[]{i, 0}, transaction.getAmount());  
12
  
13
            // One-hot encoding for labels  
14
            if (transaction.isFraudulent()) {  
15
                labels.putScalar(new int[]{i, 1}, 1.0);  
16
                labels.putScalar(new int[]{i, 0}, 0.0);  
17
            } else {  
18
                labels.putScalar(new int[]{i, 0}, 1.0);  
19
                labels.putScalar(new int[]{i, 1}, 0.0);  
20
            }  
21
        }  
22
  
23
        return dataSet = new DataSet(features, labels);  
24
    }

1
    public void trainModel(List<Transaction> transactions) {  
2
        // Shuffle the data to ensure random distribution  
3
        Collections.shuffle(transactions);  
4
  
5
        // Prepare training data  
6
        DataSet dataSet = prepareTrainingData(transactions);;  
7
  
8
        // Train the model  
9
        for (int epoch = 0; epoch < 100; epoch++) {  
10
            model.fit(dataSet);  
11
        }  
12
  
13
        System.out.println("Model trained successfully.");  
14
    }

1
public void evaluateModel(List<Transaction> transactions) {  
2
    // Split data into train and test sets  
3
    int trainSize = (int)(transactions.size() * 0.8);  
4
    List<Transaction> testSet = transactions.subList(trainSize, transactions.size());  
5
  
6
    // Prepare test data  
7
    DataSet testData = prepareTrainingData(testSet);  
8
  
9
    // Perform evaluation  
10
    Evaluation evaluation = new Evaluation(NUM_CLASSES);  
11
    INDArray predicted = model.output(testData.getFeatures());  
12
    evaluation.eval(testData.getLabels(), predicted);  
13
  
14
    // Print evaluation statistics  
15
    System.out.println(evaluation.stats());  
16
}

1
    public boolean predictFraud(Transaction transaction) {  
2
        // Convert transaction to INDArray  
3
        INDArray input = Nd4j.create(new double[][]{{transaction.getAmount()}});  
4
  
5
        // Perform prediction  
6
        INDArray output = model.output(input);  
7
  
8
        // Interpret the output  
9
        // Index 1 corresponds to fraud class (assuming one-hot encoding)
10
        return output.getDouble(0, 1) > 0.5;  
11
    }

1
package com.mongodb;  
2
  
3
import java.io.IOException;  
4
import java.util.Collections;  
5
import java.util.List;  
6
import java.util.Random;  
7
  
8
public class FraudDetectionApp { 
9
    private static FraudDetectionModel fraudDetectionModel;  
10
  
11
    public static void main(String[] args) {  
12
        MongoDBConnector mongoDBConnector = new MongoDBConnector();  
13
        TransactionRepository transactionRepository = new TransactionRepository(mongoDBConnector);  
14
        DataPreprocessor preprocessor = new DataPreprocessor(transactionRepository);  
15
  
16
        try {  
17
            // Load and prepare training data, and add to MongoDB  
18
            preprocessor.loadData("src/main/resources/creditcard.csv");  
19
            List<Transaction> transactions = transactionRepository.getAllTransactions();  
20
  
21
            // Shuffle and split data
22
            Collections.shuffle(transactions);  
23
            int trainSize = (int) (transactions.size() * 0.8);  
24
            List<Transaction> trainSet = transactions.subList(0, trainSize);  
25
            List<Transaction> testSet = transactions.subList(trainSize, transactions.size());  
26
  
27
            System.out.println("Train size: " + trainSet.size() + ", Test size: " + testSet.size());  
28
  
29
            // Create and train fraud detection model  
30
            fraudDetectionModel = new FraudDetectionModel();  
31
            fraudDetectionModel.trainModel(trainSet);  
32
  
33
            // Evaluate model  
34
            fraudDetectionModel.evaluateModel(testSet);  
35
  
36
        } catch (IOException e) {  
37
            e.printStackTrace();  
38
        }  
39
    }  
40
}

1
========================Evaluation Metrics========================
2
 # of classes:    2
3
 Accuracy:        0.8441
4
 Precision:       0.0011
5
 Recall:          0.1053
6
 F1 Score:        0.0022
7
Precision, recall & F1: reported for positive class (class 1 - "1") only

1
double calculateAccuracy(int TP, int TN, int FP, int FN) {
2
    int totalPredictions = TP + TN + FP + FN;
3
    double accuracy = (double)(TP + TN) / totalPredictions;
4
    return accuracy;
5
}

1
double calculatePrecision(int TP, int FP) {
2
    double precision = (double) TP / (TP + FP);
3
    return precision;
4
}

1
double calculateRecall(int TP, int FN) {
2
    double recall = (double) TP / (TP + FN);
3
    return recall;
4
}

1
double calculateF1Score(double precision, double recall) {
2
    double f1Score = 2 * (precision * recall) / (precision + recall);
3
    return f1Score;
4
}

1
=========================Confusion Matrix=========================
2
    0    1
3
-----------
4
 9615 1759 | 0 = 0
5
   17    2 | 1 = 1
6
7
Confusion matrix format: Actual (rowClass) predicted as (columnClass) N times
8
==================================================================

1
========================Evaluation Metrics========================
2
 # of classes:    2
3
 Accuracy:        0.9986
4
 Precision:       0.0000
5
 Recall:          0.0000
6
 F1 Score:        0.0000
7
Precision, recall & F1: reported for positive class (class 1 - "1") only
8
9
Warning: 1 class was never predicted by the model and was excluded from average precision
10
Classes excluded from average precision: [1]
11
12
=========================Confusion Matrix=========================
13
     0     1
14
-------------
15
 11377     0 | 0 = 0
16
    16     0 | 1 = 1
17
18
Confusion matrix format: Actual (rowClass) predicted as (columnClass) N times
19
==================================================================