python - My neural netwok from scratch refuse to converge, it works fine in feed forward - Stack Overflow

admin管理员组

文章数量:1186436

I have a project for a formation, I must create a neural network from scratch. My doesn't converge and I don't now why.

Here is the code :

import random
import numpy as np
import math
import joblib


class connection:
    def __init__(self, weight, neurin, neurout):
        self.weight = weight
        self.neurin = neurin  # neurone de départ
        self.neurout = neurout  # neurone d'arrivée
        self.valout = 0  # stocke la valeur de sortie
        # fait en sorte que les neurones contiennent leurs connexions
        neurin.connectout.append(self)
        neurout.connectin.append(self)

    def getout(self, val):
        self.valout = val * self.weight
        # Clipping de la valeur pour éviter des overflow
        #self.valout = np.clip(self.valout, -1e6, 1e6)  # Limiter les sorties extrêmes
        self.neurout.signal(self.valout)

    def updt(self, delta, rate, weight_clip=1):
        # Calcul de l'update de poids
        nd = delta * self.neurin.sigy  # Gradient local
        self.neurin.backprop(nd, self.weight, rate)  # Propagation vers les entrées
        # Mise à jour du poids avec clipping
        self.weight -= rate * nd
        #self.weight = np.clip(self.weight, -weight_clip, weight_clip)  # Clipping des poids


class neurone:
    def __init__(self, act, lamb, bias):
        self.connectin = []  # connexions entrantes
        self.connectout = []  # connexions sortantes
        self.bias = bias
        self.lamb = lamb
        self.act = act  # fonction d'activation
        self.sig = []  # accumulation des signaux d'entrée
        self.sigy = 0
        self.nsig = 0
        self.deltaback = 0

    def inisignal(self, signal):
        # Passe le signal dans la fonction d'activation
        sign = self.act(signal, self.bias, self.lamb)
        for c in self.connectout:
            c.getout(sign)
        self.sigy = sign

    def signal(self, signal):
        self.sig.append(signal)
        if len(self.sig) == len(self.connectin):  # Tous les signaux d'entrée sont reçus
            total_signal = sum(self.sig)
            self.sig = []
            self.nsig = total_signal
            self.inisignal(total_signal)

    def inibackprop(self, err, rate):
        De = self.act(self.nsig, self.bias, self.lamb, 1) * err
        self.deltaback = De
        # Mise à jour du biais
        self.bias -= rate * De  # Mise à jour du biais avec gradient
        for c in self.connectin:
            c.updt(De, rate)

    def backprop(self, delta, w, rate):
        self.sig.append(delta * w)
        if len(self.sig) == len(self.connectout):  # Tous les deltas sont reçus
            total_error = sum(self.sig)
            self.sig = []
            self.inibackprop(total_error, rate)


class layered_network:
    def __init__(self, nb_in, nb_out, nb_hidden, nb_perhidden, maxlamb, maxbias, maxstreigh, fonctex, foncthid):
        self.layers = []
        # Initialisation des couches d'entrée
        In = [neurone(fonctex, random.uniform(0.1, maxlamb), random.uniform(0.1, maxbias)) for _ in range(nb_in)]
        self.layers.append(In)
        # Initialisation des couches cachées
        for _ in range(nb_hidden):
            hidden_layer = [neurone(foncthid, random.uniform(0.1, maxlamb), random.uniform(0.1, maxbias))
            for _ in range(nb_perhidden)]
                self.layers.append(hidden_layer)
        # Initialisation des couches de sortie
        Out = [neurone(fonctex, random.uniform(0.1, maxlamb), random.uniform(0.1, maxbias)) for _ in range(nb_out)]
        self.layers.append(Out)
        # Création des connexions entre les couches
        for i in range(len(self.layers) - 1):
            L1, L2 = self.layers[i], self.layers[i + 1]
            for n1 in L1:
                for n2 in L2:
                    C = connection(random.uniform(-maxstreigh, maxstreigh), n1, n2)

    def getoutput(self, Input):
        for i, val in enumerate(Input):
            self.layers[0][i].inisignal(val)
        return [n.sigy for n in self.layers[-1]]

    def onelearn(self, Input, expected, rate):
        output = self.getoutput(Input)
        dif = np.array(output) - np.array(expected)
        for i, n in enumerate(self.layers[-1]):
            n.inibackprop(dif[i], rate)

    def multilearn(self, Input, expected, rate):
        for i, I in enumerate(Input):
            self.onelearn(I, expected[i], rate)

    def cyclelearn(self, Input, expected, rate, nb):
        for i in range(nb):
            # Décroissant cosinusoïdal
            self.multilearn(Input, expected, rate*abs(i/30))

    def moyError(self, Input, expected):
        R = []
            for i in range(len(Input)):
            output = self.getoutput(Input[i])
            err = np.average(np.abs(np.array(output) - np.array(expected[i])))
            R.append(err)
        return np.average(R)

    def save(self, dir):
        saveobject(dir, self.layers)

    def load(self, dir):
        self.layers = loadobject(dir)


def Csigmoid(signal, bias, lamb, mod=0):
    max_input = 500
    input_val = -(signal + bias) * lamb
    input_val = np.clip(input_val, -max_input, max_input)
    n = np.exp(input_val)
    if mod == 0:
        return 100 / (1 + n)
    elif mod == 1:
        sigmoid_val = 100 / (1 + n)
        return lamb * n * (sigmoid_val ** 2)


def tanh(signal, bias, lamb, mod=0):
    max_input = 100
    input_val = (signal + bias) * lamb
    input_val = np.clip(input_val, -max_input, max_input)
    if mod == 0:
        return np.tanh(input_val)
    elif mod == 1:
        return lamb * (1 - np.tanh(input_val) ** 2)


def relu(signal, bias, lamb, mod=0):
    if mod == 0:
        a = signal + bias * lamb
        return a if a > 0 else 0
    if mod == 1:
        return lamb if signal + bias > 0 else 0


def Lrelu(signal, bias, lamb, mod=0):
    if mod == 0:
        return lamb * (signal + bias) if signal + bias > 0 else (signal + bias) / lamb
    if mod == 1:
        return lamb if signal + bias > 0 else 1 / lamb


def saveobject(Dir, obj):
    with open(Dir, "wb") as F:
        joblib.dump(obj, F)


def loadobject(Dir):
    with open(Dir, "rb") as F:
        return joblib.load(F)

I have tried to change the formulas, clipping the value, changing the activation function, and it still doesn't work..so I am lost right now

I want the neural network to get closer to an expected value, assosiated to each inputs. I have tried to change the activation function, I have try to limit the range of value, all failed.

本文标签：