Java源码示例：org.nd4j.linalg.api.ops.impl.broadcast.BroadcastMulOp
示例1
/**
* Denormalize a data array
*
* @param array the data to denormalize
* @param stats statistics of the data population
*/
@Override
public void revert(INDArray array, INDArray maskArray, MinMaxStats stats) {
// Subtract target range minimum value
array.subi(minRange);
// Scale by target range
array.divi(maxRange - minRange);
if (array.rank() <= 2) {
array.muliRowVector(stats.getRange());
array.addiRowVector(stats.getLower());
} else {
Nd4j.getExecutioner().execAndReturn(new BroadcastMulOp(array, stats.getRange(), array, 1));
Nd4j.getExecutioner().execAndReturn(new BroadcastAddOp(array, stats.getLower(), array, 1));
}
if (maskArray != null) {
DataSetUtil.setMaskedValuesToZero(array, maskArray);
}
}
示例2
/**
* Denormalize a data array
*
* @param array the data to denormalize
* @param stats statistics of the data population
*/
@Override
public void revert(INDArray array, INDArray maskArray, MinMaxStats stats) {
// Subtract target range minimum value
array.subi(minRange);
// Scale by target range
array.divi(maxRange - minRange);
if (array.rank() <= 2) {
array.muliRowVector(stats.getRange());
array.addiRowVector(stats.getLower());
} else {
Nd4j.getExecutioner().execAndReturn(new BroadcastMulOp(array, stats.getRange().castTo(array.dataType()), array, 1));
Nd4j.getExecutioner().execAndReturn(new BroadcastAddOp(array, stats.getLower().castTo(array.dataType()), array, 1));
}
if (maskArray != null) {
DataSetUtil.setMaskedValuesToZero(array, maskArray);
}
}
示例3
@Test
public void testBroadcastMultiDim() throws Exception {
//Broadcast 1d: OK
INDArray arr2d = Nd4j.ones(2,3);
INDArray toBCRow = Nd4j.create(new double[]{1,0,0});
Nd4j.getExecutioner().exec(new BroadcastMulOp(arr2d, toBCRow, arr2d, 1));
INDArray exp2d = Nd4j.create(
new double[][]{
{1,0,0},
{1,0,0}});
assertEquals(exp2d, arr2d);
//Broadcast 2d on 3d:
INDArray arr3d = Nd4j.ones(2,3,5);
INDArray bc2d = Nd4j.create(new double[][]{
{1,1,1,1,1},
{1,1,1,0,0}});
bc2d.get(NDArrayIndex.point(1), NDArrayIndex.interval(3,5)).assign(0);
Nd4j.getExecutioner().exec(new BroadcastMulOp(arr3d, bc2d, arr3d, 0, 2));
INDArray exp3d = Nd4j.ones(2,3,5);
exp3d.get(NDArrayIndex.point(1), NDArrayIndex.all(), NDArrayIndex.interval(3,5)).assign(0);
for( int i=0; i<2; i++ ){
System.out.println("Arr - " + i);
System.out.println(arr3d.get(NDArrayIndex.point(i), NDArrayIndex.all(), NDArrayIndex.all()));
System.out.println("Exp - " + i);
System.out.println(exp3d.get(NDArrayIndex.point(i), NDArrayIndex.all(), NDArrayIndex.all()));
System.out.println();
}
assertEquals(exp3d, arr3d);
}
示例4
@Test
public void testBroadcastMultiDim() {
INDArray data = Nd4j.linspace(1, 30, 30).reshape(2, 3, 5);
System.out.println(data);
INDArray mask = Nd4j.create(new double[][] {{1.00, 1.00, 1.00, 1.00, 1.00}, {1.00, 1.00, 1.00, 0.00, 0.00}});
Nd4j.getExecutioner().exec(new BroadcastMulOp(data, mask, data, 0, 2));
INDArray assertion = Nd4j.create(new double[] {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0,
13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 0.0, 0.0, 21.0, 22.0, 23.0, 0.0, 0.0, 26.0, 27.0, 28.0, 0.0,
0.0}).reshape(2, 3, 5);
assertEquals(assertion, data);
}
示例5
/**
* Denormalize a data array
*
* @param array the data to denormalize
* @param stats statistics of the data population
*/
@Override
public void revert(INDArray array, INDArray maskArray, DistributionStats stats) {
if (array.rank() <= 2) {
array.muliRowVector(filteredStd(stats));
array.addiRowVector(stats.getMean());
} else {
Nd4j.getExecutioner().execAndReturn(new BroadcastMulOp(array, filteredStd(stats), array, 1));
Nd4j.getExecutioner().execAndReturn(new BroadcastAddOp(array, stats.getMean(), array, 1));
}
if (maskArray != null) {
DataSetUtil.setMaskedValuesToZero(array, maskArray);
}
}
示例6
@Test
public void testBroadcastMultiDim() {
INDArray data = Nd4j.linspace(1, 30, 30, DataType.DOUBLE).reshape(2, 3, 5);
// System.out.println(data);
INDArray mask = Nd4j.create(new double[][] {{1.00, 1.00, 1.00, 1.00, 1.00}, {1.00, 1.00, 1.00, 0.00, 0.00}});
Nd4j.getExecutioner().exec(new BroadcastMulOp(data, mask, data, 0, 2));
INDArray assertion = Nd4j.create(new double[] {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0,
13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 0.0, 0.0, 21.0, 22.0, 23.0, 0.0, 0.0, 26.0, 27.0, 28.0, 0.0,
0.0}).reshape(2, 3, 5);
assertEquals(assertion, data);
}
示例7
/**
* Denormalize a data array
*
* @param array the data to denormalize
* @param stats statistics of the data population
*/
@Override
public void revert(INDArray array, INDArray maskArray, DistributionStats stats) {
if (array.rank() <= 2) {
array.muliRowVector(filteredStd(stats));
array.addiRowVector(stats.getMean());
} else {
Nd4j.getExecutioner().execAndReturn(new BroadcastMulOp(array, filteredStd(stats).castTo(array.dataType()), array, 1));
Nd4j.getExecutioner().execAndReturn(new BroadcastAddOp(array, stats.getMean().castTo(array.dataType()), array, 1));
}
if (maskArray != null) {
DataSetUtil.setMaskedValuesToZero(array, maskArray);
}
}
示例8
@Override
public Pair<Gradient, INDArray[]> doBackward(boolean tbptt, LayerWorkspaceMgr workspaceMgr) {
if (!canDoBackward())
throw new IllegalStateException("Cannot do backward pass: error not set");
INDArray a = inputs[0];
INDArray b = inputs[1];
INDArray out = doForward(tbptt, workspaceMgr);
Transforms.max(out, eps, false); // in case of 0
INDArray dLdlambda = epsilon; //dL/dlambda aka 'epsilon' - from layer above
INDArray sNegHalf = out.rdiv(1.0); //s^(-1/2) = 1.0 / s^(1/2) = 1.0 / out
INDArray diff;
try(MemoryWorkspace ws = workspaceMgr.notifyScopeBorrowed(ArrayType.ACTIVATION_GRAD)){
diff = a.sub(b);
}
INDArray first = dLdlambda.mul(sNegHalf); //Column vector for all cases
INDArray dLda;
INDArray dLdb;
if (a.rank() == 2) {
//2d case (MLPs etc)
dLda = diff.muliColumnVector(first);
try(MemoryWorkspace ws = workspaceMgr.notifyScopeBorrowed(ArrayType.ACTIVATION_GRAD)) {
dLdb = dLda.neg();
}
} else {
//RNN and CNN case - Broadcast along dimension 0
dLda = Nd4j.getExecutioner().exec(new BroadcastMulOp(diff, first, diff, 0));
try(MemoryWorkspace ws = workspaceMgr.notifyScopeBorrowed(ArrayType.ACTIVATION_GRAD)) {
dLdb = dLda.neg();
}
}
return new Pair<>(null, new INDArray[] {dLda, dLdb});
}
示例9
public static void setMaskedValuesToZero(INDArray data, INDArray mask) {
if (mask == null || data.rank() != 3)
return;
Nd4j.getExecutioner().exec(new BroadcastMulOp(data, mask, data, 0, 2));
}
示例10
public static void setMaskedValuesToZero(INDArray data, INDArray mask) {
if (mask == null || data.rank() != 3)
return;
Nd4j.getExecutioner().exec(new BroadcastMulOp(data, mask, data, 0, 2));
}
示例11
private INDArray epsilonHelperFullArray(INDArray inputArray, INDArray epsilon, int[] poolDim) {
//Broadcast: occurs on the remaining dimensions, after the pool dimensions have been removed.
//TODO find a more efficient way to do this
int[] broadcastDims = new int[inputArray.rank() - poolDim.length];
int count = 0;
for (int i = 0; i < inputArray.rank(); i++) {
if (ArrayUtils.contains(poolDim, i))
continue;
broadcastDims[count++] = i;
}
switch (poolingType) {
case MAX:
INDArray isMax = Nd4j.exec(new IsMax(inputArray, inputArray.ulike(), poolDim))[0];
return Nd4j.getExecutioner().exec(new BroadcastMulOp(isMax, epsilon, isMax, broadcastDims));
case AVG:
//if out = avg(in,dims) then dL/dIn = 1/N * dL/dOut
int n = 1;
for (int d : poolDim) {
n *= inputArray.size(d);
}
INDArray ret = inputArray.ulike();
Nd4j.getExecutioner().exec(new BroadcastCopyOp(ret, epsilon, ret, broadcastDims));
ret.divi(n);
return ret;
case SUM:
INDArray retSum = inputArray.ulike();
Nd4j.getExecutioner().exec(new BroadcastCopyOp(retSum, epsilon, retSum, broadcastDims));
return retSum;
case PNORM:
int pnorm = layerConf().getPnorm();
//First: do forward pass to get pNorm array
INDArray abs = Transforms.abs(inputArray, true);
Transforms.pow(abs, pnorm, false);
INDArray pNorm = Transforms.pow(abs.sum(poolDim), 1.0 / pnorm);
//dL/dIn = dL/dOut * dOut/dIn
//dOut/dIn = in .* |in|^(p-2) / ||in||_p^(p-1), where ||in||_p is the output p-norm
INDArray numerator;
if (pnorm == 2) {
numerator = inputArray.dup();
} else {
INDArray absp2 = Transforms.pow(Transforms.abs(inputArray, true), pnorm - 2, false);
numerator = inputArray.mul(absp2);
}
INDArray denom = Transforms.pow(pNorm, pnorm - 1, false);
denom.rdivi(epsilon);
Nd4j.getExecutioner().execAndReturn(new BroadcastMulOp(numerator, denom, numerator, broadcastDims));
return numerator;
default:
throw new RuntimeException("Unknown or not supported pooling type: " + poolingType + " " + layerId());
}
}
示例12
public static INDArray maskedPoolingTimeSeries(PoolingType poolingType, INDArray toReduce, INDArray mask,
int pnorm, DataType dataType) {
if (toReduce.rank() != 3) {
throw new IllegalArgumentException("Expect rank 3 array: got " + toReduce.rank());
}
if (mask.rank() != 2) {
throw new IllegalArgumentException("Expect rank 2 array for mask: got " + mask.rank());
}
toReduce = toReduce.castTo(dataType);
mask = mask.castTo(dataType);
//Sum pooling: easy. Multiply by mask, then sum as normal
//Average pooling: as above, but do a broadcast element-wise divi by mask.sum(1)
//Max pooling: set to -inf if mask is 0, then do max as normal
switch (poolingType) {
case MAX:
INDArray negInfMask = mask.castTo(dataType).rsub(1.0);
BooleanIndexing.replaceWhere(negInfMask, Double.NEGATIVE_INFINITY, Conditions.equals(1.0));
INDArray withInf = Nd4j.createUninitialized(dataType, toReduce.shape());
Nd4j.getExecutioner().exec(new BroadcastAddOp(toReduce, negInfMask, withInf, 0, 2));
//At this point: all the masked out steps have value -inf, hence can't be the output of the MAX op
return withInf.max(2);
case AVG:
case SUM:
INDArray masked = Nd4j.createUninitialized(dataType, toReduce.shape());
Nd4j.getExecutioner().exec(new BroadcastMulOp(toReduce, mask, masked, 0, 2));
INDArray summed = masked.sum(2);
if (poolingType == PoolingType.SUM) {
return summed;
}
INDArray maskCounts = mask.sum(1);
summed.diviColumnVector(maskCounts);
return summed;
case PNORM:
//Similar to average and sum pooling: there's no N term here, so we can just set the masked values to 0
INDArray masked2 = Nd4j.createUninitialized(dataType, toReduce.shape());
Nd4j.getExecutioner().exec(new BroadcastMulOp(toReduce, mask, masked2, 0, 2));
INDArray abs = Transforms.abs(masked2, true);
Transforms.pow(abs, pnorm, false);
INDArray pNorm = abs.sum(2);
return Transforms.pow(pNorm, 1.0 / pnorm);
default:
throw new UnsupportedOperationException("Unknown or not supported pooling type: " + poolingType);
}
}
示例13
public static INDArray maskedPoolingEpsilonTimeSeries(PoolingType poolingType, INDArray input, INDArray mask,
INDArray epsilon2d, int pnorm) {
if (input.rank() != 3) {
throw new IllegalArgumentException("Expect rank 3 input activation array: got " + input.rank());
}
if (mask.rank() != 2) {
throw new IllegalArgumentException("Expect rank 2 array for mask: got " + mask.rank());
}
if (epsilon2d.rank() != 2) {
throw new IllegalArgumentException("Expected rank 2 array for errors: got " + epsilon2d.rank());
}
//Mask: [minibatch, tsLength]
//Epsilon: [minibatch, vectorSize]
mask = mask.castTo(input.dataType());
switch (poolingType) {
case MAX:
INDArray negInfMask = mask.rsub(1.0);
BooleanIndexing.replaceWhere(negInfMask, Double.NEGATIVE_INFINITY, Conditions.equals(1.0));
INDArray withInf = Nd4j.createUninitialized(input.dataType(), input.shape());
Nd4j.getExecutioner().exec(new BroadcastAddOp(input, negInfMask, withInf, 0, 2));
//At this point: all the masked out steps have value -inf, hence can't be the output of the MAX op
INDArray isMax = Nd4j.exec(new IsMax(withInf, withInf.ulike(), 2))[0];
return Nd4j.getExecutioner().exec(new BroadcastMulOp(isMax, epsilon2d, isMax, 0, 1));
case AVG:
case SUM:
//if out = sum(in,dims) then dL/dIn = dL/dOut -> duplicate to each step and mask
//if out = avg(in,dims) then dL/dIn = 1/N * dL/dOut
//With masking: N differs for different time series
INDArray out = Nd4j.createUninitialized(input.dataType(), input.shape(), 'f');
//Broadcast copy op, then divide and mask to 0 as appropriate
Nd4j.getExecutioner().exec(new BroadcastCopyOp(out, epsilon2d, out, 0, 1));
Nd4j.getExecutioner().exec(new BroadcastMulOp(out, mask, out, 0, 2));
if (poolingType == PoolingType.SUM) {
return out;
}
INDArray nEachTimeSeries = mask.sum(1); //[minibatchSize,tsLength] -> [minibatchSize,1]
Nd4j.getExecutioner().exec(new BroadcastDivOp(out, nEachTimeSeries, out, 0));
return out;
case PNORM:
//Similar to average and sum pooling: there's no N term here, so we can just set the masked values to 0
INDArray masked2 = Nd4j.createUninitialized(input.dataType(), input.shape());
Nd4j.getExecutioner().exec(new BroadcastMulOp(input, mask, masked2, 0, 2));
INDArray abs = Transforms.abs(masked2, true);
Transforms.pow(abs, pnorm, false);
INDArray pNorm = Transforms.pow(abs.sum(2), 1.0 / pnorm);
INDArray numerator;
if (pnorm == 2) {
numerator = input.dup();
} else {
INDArray absp2 = Transforms.pow(Transforms.abs(input, true), pnorm - 2, false);
numerator = input.mul(absp2);
}
INDArray denom = Transforms.pow(pNorm, pnorm - 1, false);
denom.rdivi(epsilon2d);
Nd4j.getExecutioner().execAndReturn(new BroadcastMulOp(numerator, denom, numerator, 0, 1));
Nd4j.getExecutioner().exec(new BroadcastMulOp(numerator, mask, numerator, 0, 2)); //Apply mask
return numerator;
default:
throw new UnsupportedOperationException("Unknown or not supported pooling type: " + poolingType);
}
}
示例14
public static INDArray maskedPoolingConvolution(PoolingType poolingType, INDArray toReduce, INDArray mask, int pnorm, DataType dataType) {
if(mask.rank() != 4){
//TODO BETTER ERROR MESSAGE EXPLAINING FORMAT
//TODO ALSO HANDLE LEGACY FORMAT WITH WARNING WHERE POSSIBLE
throw new IllegalStateException("Expected rank 4 mask array: Got array with shape " + Arrays.toString(mask.shape()));
}
mask = mask.castTo(dataType); //no-op if already correct dtype
// [minibatch, channels, h, w] data with a mask array of shape [minibatch, 1, X, Y]
// where X=(1 or inH) and Y=(1 or inW)
//General case: must be equal or 1 on each dimension
int[] dimensions = new int[4];
int count = 0;
for(int i=0; i<4; i++ ){
if(toReduce.size(i) == mask.size(i)){
dimensions[count++] = i;
}
}
if(count < 4){
dimensions = Arrays.copyOfRange(dimensions, 0, count);
}
switch (poolingType) {
case MAX:
//TODO This is ugly - replace it with something better... Need something like a Broadcast CAS op
INDArray negInfMask;
if(mask.dataType() == DataType.BOOL){
negInfMask = Transforms.not(mask).castTo(dataType);
} else {
negInfMask = mask.rsub(1.0);
}
BooleanIndexing.replaceWhere(negInfMask, Double.NEGATIVE_INFINITY, Conditions.equals(1.0));
INDArray withInf = Nd4j.createUninitialized(dataType, toReduce.shape());
Nd4j.getExecutioner().exec(new BroadcastAddOp(toReduce, negInfMask, withInf, dimensions));
//At this point: all the masked out steps have value -inf, hence can't be the output of the MAX op
return withInf.max(2, 3);
case AVG:
case SUM:
INDArray masked = Nd4j.createUninitialized(dataType, toReduce.shape());
Nd4j.getExecutioner().exec(new BroadcastMulOp(toReduce, mask, masked, dimensions));
INDArray summed = masked.sum(2, 3);
if (poolingType == PoolingType.SUM) {
return summed;
}
INDArray maskCounts = mask.sum(1,2,3);
summed.diviColumnVector(maskCounts);
return summed;
case PNORM:
//Similar to average and sum pooling: there's no N term here, so we can just set the masked values to 0
INDArray masked2 = Nd4j.createUninitialized(dataType, toReduce.shape());
Nd4j.getExecutioner().exec(new BroadcastMulOp(toReduce, mask, masked2, dimensions));
INDArray abs = Transforms.abs(masked2, true);
Transforms.pow(abs, pnorm, false);
INDArray pNorm = abs.sum(2, 3);
return Transforms.pow(pNorm, 1.0 / pnorm);
default:
throw new UnsupportedOperationException("Unknown or not supported pooling type: " + poolingType);
}
}
示例15
public static INDArray maskedPoolingEpsilonCnn(PoolingType poolingType, INDArray input, INDArray mask,
INDArray epsilon2d, int pnorm, DataType dataType) {
// [minibatch, channels, h=1, w=X] or [minibatch, channels, h=X, w=1] data
// with a mask array of shape [minibatch, X]
//If masking along height: broadcast dimensions are [0,2]
//If masking along width: broadcast dimensions are [0,3]
mask = mask.castTo(dataType); //No-op if correct type
//General case: must be equal or 1 on each dimension
int[] dimensions = new int[4];
int count = 0;
for(int i=0; i<4; i++ ){
if(input.size(i) == mask.size(i)){
dimensions[count++] = i;
}
}
if(count < 4){
dimensions = Arrays.copyOfRange(dimensions, 0, count);
}
switch (poolingType) {
case MAX:
//TODO This is ugly - replace it with something better... Need something like a Broadcast CAS op
INDArray negInfMask;
if(mask.dataType() == DataType.BOOL){
negInfMask = Transforms.not(mask).castTo(dataType);
} else {
negInfMask = mask.rsub(1.0);
}
BooleanIndexing.replaceWhere(negInfMask, Double.NEGATIVE_INFINITY, Conditions.equals(1.0));
INDArray withInf = Nd4j.createUninitialized(dataType, input.shape());
Nd4j.getExecutioner().exec(new BroadcastAddOp(input, negInfMask, withInf, dimensions));
//At this point: all the masked out steps have value -inf, hence can't be the output of the MAX op
INDArray isMax = Nd4j.exec(new IsMax(withInf, withInf.ulike(), 2, 3))[0];
return Nd4j.getExecutioner().exec(new BroadcastMulOp(isMax, epsilon2d, isMax, 0, 1));
case AVG:
case SUM:
//if out = sum(in,dims) then dL/dIn = dL/dOut -> duplicate to each step and mask
//if out = avg(in,dims) then dL/dIn = 1/N * dL/dOut
//With masking: N differs for different time series
INDArray out = Nd4j.createUninitialized(dataType, input.shape(), 'f');
//Broadcast copy op, then divide and mask to 0 as appropriate
Nd4j.getExecutioner().exec(new BroadcastCopyOp(out, epsilon2d, out, 0, 1));
Nd4j.getExecutioner().exec(new BroadcastMulOp(out, mask, out, dimensions));
if (poolingType == PoolingType.SUM) {
return out;
}
//Note that with CNNs, current design is restricted to [minibatch, channels, 1, W] ot [minibatch, channels, H, 1]
INDArray nEachTimeSeries = mask.sum(1,2,3); //[minibatchSize,tsLength] -> [minibatchSize,1]
Nd4j.getExecutioner().exec(new BroadcastDivOp(out, nEachTimeSeries, out, 0));
return out;
case PNORM:
//Similar to average and sum pooling: there's no N term here, so we can just set the masked values to 0
INDArray masked2 = Nd4j.createUninitialized(dataType, input.shape());
Nd4j.getExecutioner().exec(new BroadcastMulOp(input, mask, masked2, dimensions));
INDArray abs = Transforms.abs(masked2, true);
Transforms.pow(abs, pnorm, false);
INDArray pNorm = Transforms.pow(abs.sum(2, 3), 1.0 / pnorm);
INDArray numerator;
if (pnorm == 2) {
numerator = input.dup();
} else {
INDArray absp2 = Transforms.pow(Transforms.abs(input, true), pnorm - 2, false);
numerator = input.mul(absp2);
}
INDArray denom = Transforms.pow(pNorm, pnorm - 1, false);
denom.rdivi(epsilon2d);
Nd4j.getExecutioner().execAndReturn(new BroadcastMulOp(numerator, denom, numerator, 0, 1));
Nd4j.getExecutioner().exec(new BroadcastMulOp(numerator, mask, numerator, dimensions)); //Apply mask
return numerator;
default:
throw new UnsupportedOperationException("Unknown or not supported pooling type: " + poolingType);
}
}
示例16
@Test
public void testCnnForwardBackward() {
double eps = 1e-5;
int nIn = 4;
int hw = 3;
int minibatch = 2;
Nd4j.getRandom().setSeed(12345);
INDArray input = Nd4j.rand('c', new int[]{minibatch, nIn, hw, hw});
//TODO: other values for gamma/beta
INDArray gamma = Nd4j.ones(1, nIn);
INDArray beta = Nd4j.zeros(1, nIn);
Layer l = getLayer(nIn, eps, false, -1, -1);
INDArray mean = input.mean(0, 2, 3);
INDArray var = input.var(false, 0, 2, 3);
INDArray xHat = Nd4j.getExecutioner().exec(new BroadcastSubOp(input, mean, input.dup(), 1));
Nd4j.getExecutioner().exec(new BroadcastDivOp(xHat, Transforms.sqrt(var.add(eps), true), xHat, 1));
INDArray outExpected = Nd4j.getExecutioner().exec(new BroadcastMulOp(xHat, gamma, xHat.dup(), 1));
Nd4j.getExecutioner().exec(new BroadcastAddOp(outExpected, beta, outExpected, 1));
INDArray out = l.activate(input, true, LayerWorkspaceMgr.noWorkspaces());
// System.out.println(Arrays.toString(outExpected.data().asDouble()));
// System.out.println(Arrays.toString(out.data().asDouble()));
assertEquals(outExpected, out);
//-------------------------------------------------------------
//Check backprop
INDArray epsilon = Nd4j.rand('c', new int[]{minibatch, nIn, hw, hw}); //dL/dy
int effectiveMinibatch = minibatch * hw * hw;
INDArray dldgammaExp = epsilon.mul(xHat).sum(0, 2, 3);
dldgammaExp = dldgammaExp.reshape(1, dldgammaExp.length());
INDArray dldbetaExp = epsilon.sum(0, 2, 3);
dldbetaExp = dldbetaExp.reshape(1, dldbetaExp.length());
INDArray dldxhat = Nd4j.getExecutioner().exec(new BroadcastMulOp(epsilon, gamma, epsilon.dup(), 1)); //epsilon.mulRowVector(gamma);
INDArray inputSubMean = Nd4j.getExecutioner().exec(new BroadcastSubOp(input, mean, input.dup(), 1));
INDArray dldvar = dldxhat.mul(inputSubMean).mul(-0.5);
dldvar = Nd4j.getExecutioner().exec(
new BroadcastMulOp(dldvar, Transforms.pow(var.add(eps), -3.0 / 2.0, true), dldvar.dup(), 1));
dldvar = dldvar.sum(0, 2, 3);
INDArray dldmu = Nd4j
.getExecutioner().exec(new BroadcastMulOp(dldxhat,
Transforms.pow(var.add(eps), -1.0 / 2.0, true), dldxhat.dup(), 1))
.neg().sum(0, 2, 3);
dldmu = dldmu.add(dldvar.mul(inputSubMean.mul(-2.0).sum(0, 2, 3).div(effectiveMinibatch)));
INDArray dldinExp = Nd4j.getExecutioner().exec(
new BroadcastMulOp(dldxhat, Transforms.pow(var.add(eps), -1.0 / 2.0, true), dldxhat.dup(), 1));
dldinExp = dldinExp.add(Nd4j.getExecutioner().exec(
new BroadcastMulOp(inputSubMean.mul(2.0 / effectiveMinibatch), dldvar, inputSubMean.dup(), 1)));
dldinExp = Nd4j.getExecutioner().exec(
new BroadcastAddOp(dldinExp, dldmu.mul(1.0 / effectiveMinibatch), dldinExp.dup(), 1));
Pair<Gradient, INDArray> p = l.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces());
INDArray dldgamma = p.getFirst().getGradientFor("gamma");
INDArray dldbeta = p.getFirst().getGradientFor("beta");
assertEquals(dldgammaExp, dldgamma);
assertEquals(dldbetaExp, dldbeta);
// System.out.println("EPSILONS");
// System.out.println(Arrays.toString(dldinExp.data().asDouble()));
// System.out.println(Arrays.toString(p.getSecond().dup().data().asDouble()));
assertEquals(dldinExp, p.getSecond());
}
示例17
@Test
public void testMaskingCnnDim3_SingleExample() {
//Test masking, where mask is along dimension 3
int minibatch = 1;
int depthIn = 2;
int depthOut = 2;
int nOut = 2;
int height = 3;
int width = 6;
PoolingType[] poolingTypes =
new PoolingType[] {PoolingType.SUM, PoolingType.AVG, PoolingType.MAX, PoolingType.PNORM};
for (PoolingType pt : poolingTypes) {
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER)
.convolutionMode(ConvolutionMode.Same).seed(12345L).list()
.layer(0, new ConvolutionLayer.Builder().nIn(depthIn).nOut(depthOut).kernelSize(height, 2)
.stride(height, 1).activation(Activation.TANH).build())
.layer(1, new org.deeplearning4j.nn.conf.layers.GlobalPoolingLayer.Builder().poolingType(pt)
.build())
.layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
.activation(Activation.SOFTMAX).nIn(depthOut).nOut(nOut).build())
.build();
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
INDArray inToBeMasked = Nd4j.rand(new int[] {minibatch, depthIn, height, width});
//Shape for mask: [minibatch, 1, 1, width]
INDArray maskArray = Nd4j.create(new double[] {1, 1, 1, 1, 1, 0}, new int[]{1,1,1,width});
//Multiply the input by the mask array, to ensure the 0s in the mask correspond to 0s in the input vector
// as would be the case in practice...
Nd4j.getExecutioner().exec(new BroadcastMulOp(inToBeMasked, maskArray, inToBeMasked, 0, 3));
net.setLayerMaskArrays(maskArray, null);
INDArray outMasked = net.output(inToBeMasked);
net.clearLayerMaskArrays();
int numSteps = width - 1;
INDArray subset = inToBeMasked.get(NDArrayIndex.interval(0, 0, true), NDArrayIndex.all(),
NDArrayIndex.all(), NDArrayIndex.interval(0, numSteps));
assertArrayEquals(new long[] {1, depthIn, height, 5}, subset.shape());
INDArray outSubset = net.output(subset);
INDArray outMaskedSubset = outMasked.getRow(0);
assertEquals(outSubset, outMaskedSubset);
//Finally: check gradient calc for exceptions
net.setLayerMaskArrays(maskArray, null);
net.setInput(inToBeMasked);
INDArray labels = Nd4j.create(new double[] {0, 1}, new long[]{1,2});
net.setLabels(labels);
net.computeGradientAndScore();
}
}
示例18
@Test
public void testMaskingCnnDim2_SingleExample() {
//Test masking, where mask is along dimension 2
int minibatch = 1;
int depthIn = 2;
int depthOut = 2;
int nOut = 2;
int height = 6;
int width = 3;
PoolingType[] poolingTypes =
new PoolingType[] {PoolingType.SUM, PoolingType.AVG, PoolingType.MAX, PoolingType.PNORM};
for (PoolingType pt : poolingTypes) {
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER)
.convolutionMode(ConvolutionMode.Same).seed(12345L).list()
.layer(0, new ConvolutionLayer.Builder().nIn(depthIn).nOut(depthOut).kernelSize(2, width)
.stride(1, width).activation(Activation.TANH).build())
.layer(1, new org.deeplearning4j.nn.conf.layers.GlobalPoolingLayer.Builder().poolingType(pt)
.build())
.layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
.activation(Activation.SOFTMAX).nIn(depthOut).nOut(nOut).build())
.build();
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
INDArray inToBeMasked = Nd4j.rand(new int[] {minibatch, depthIn, height, width});
//Shape for mask: [minibatch, width]
INDArray maskArray = Nd4j.create(new double[] {1, 1, 1, 1, 1, 0}, new int[]{1,1,height,1});
//Multiply the input by the mask array, to ensure the 0s in the mask correspond to 0s in the input vector
// as would be the case in practice...
Nd4j.getExecutioner().exec(new BroadcastMulOp(inToBeMasked, maskArray, inToBeMasked, 0, 2));
net.setLayerMaskArrays(maskArray, null);
INDArray outMasked = net.output(inToBeMasked);
net.clearLayerMaskArrays();
int numSteps = height - 1;
INDArray subset = inToBeMasked.get(NDArrayIndex.interval(0, 0, true), NDArrayIndex.all(),
NDArrayIndex.interval(0, numSteps), NDArrayIndex.all());
assertArrayEquals(new long[] {1, depthIn, 5, width}, subset.shape());
INDArray outSubset = net.output(subset);
INDArray outMaskedSubset = outMasked.getRow(0);
assertEquals(outSubset, outMaskedSubset);
//Finally: check gradient calc for exceptions
net.setLayerMaskArrays(maskArray, null);
net.setInput(inToBeMasked);
INDArray labels = Nd4j.create(new double[] {0, 1}, new long[]{1,2});
net.setLabels(labels);
net.computeGradientAndScore();
}
}
示例19
@Test
public void testMaskingCnnDim3() {
//Test masking, where mask is along dimension 3
int minibatch = 3;
int depthIn = 3;
int depthOut = 4;
int nOut = 5;
int height = 3;
int width = 6;
PoolingType[] poolingTypes =
new PoolingType[] {PoolingType.SUM, PoolingType.AVG, PoolingType.MAX, PoolingType.PNORM};
for (PoolingType pt : poolingTypes) {
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER)
.convolutionMode(ConvolutionMode.Same).seed(12345L).list()
.layer(0, new ConvolutionLayer.Builder().nIn(depthIn).nOut(depthOut).kernelSize(height, 2)
.stride(height, 1).activation(Activation.TANH).build())
.layer(1, new org.deeplearning4j.nn.conf.layers.GlobalPoolingLayer.Builder().poolingType(pt)
.build())
.layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
.activation(Activation.SOFTMAX).nIn(depthOut).nOut(nOut).build())
.build();
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
INDArray inToBeMasked = Nd4j.rand(new int[] {minibatch, depthIn, height, width});
//Shape for mask: [minibatch, width]
INDArray maskArray = Nd4j.create(new double[][] {{1, 1, 1, 1, 1, 1}, {1, 1, 1, 1, 1, 0}, {1, 1, 1, 1, 0, 0}})
.reshape('c', minibatch, 1, 1, width);
//Multiply the input by the mask array, to ensure the 0s in the mask correspond to 0s in the input vector
// as would be the case in practice...
Nd4j.getExecutioner().exec(new BroadcastMulOp(inToBeMasked, maskArray, inToBeMasked, 0, 3));
net.setLayerMaskArrays(maskArray, null);
INDArray outMasked = net.output(inToBeMasked);
net.clearLayerMaskArrays();
for (int i = 0; i < minibatch; i++) {
int numSteps = width - i;
INDArray subset = inToBeMasked.get(NDArrayIndex.interval(i, i, true), NDArrayIndex.all(),
NDArrayIndex.all(), NDArrayIndex.interval(0, numSteps));
assertArrayEquals(new long[] {1, depthIn, height, width - i}, subset.shape());
INDArray outSubset = net.output(subset);
INDArray outMaskedSubset = outMasked.getRow(i, true);
assertEquals("minibatch: " + i, outSubset, outMaskedSubset);
}
}
}
示例20
@Test
public void testMaskingCnnDim2() {
//Test masking, where mask is along dimension 2
int minibatch = 3;
int depthIn = 3;
int depthOut = 4;
int nOut = 5;
int height = 5;
int width = 4;
PoolingType[] poolingTypes =
new PoolingType[] {PoolingType.SUM, PoolingType.AVG, PoolingType.MAX, PoolingType.PNORM};
for (PoolingType pt : poolingTypes) {
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER)
.convolutionMode(ConvolutionMode.Same).seed(12345L).list()
.layer(0, new ConvolutionLayer.Builder().nIn(depthIn).nOut(depthOut).kernelSize(2, width)
.stride(1, width).activation(Activation.TANH).build())
.layer(1, new org.deeplearning4j.nn.conf.layers.GlobalPoolingLayer.Builder().poolingType(pt)
.build())
.layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
.activation(Activation.SOFTMAX).nIn(depthOut).nOut(nOut).build())
.build();
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
INDArray inToBeMasked = Nd4j.rand(new int[] {minibatch, depthIn, height, width});
//Shape for mask: [minibatch, 1, height, 1] -> broadcast
INDArray maskArray = Nd4j.create(new double[][] {{1, 1, 1, 1, 1}, {1, 1, 1, 1, 0}, {1, 1, 1, 0, 0}})
.reshape('c', minibatch, 1, height, 1);
//Multiply the input by the mask array, to ensure the 0s in the mask correspond to 0s in the input vector
// as would be the case in practice...
Nd4j.getExecutioner().exec(new BroadcastMulOp(inToBeMasked, maskArray, inToBeMasked, 0, 2));
net.setLayerMaskArrays(maskArray, null);
INDArray outMasked = net.output(inToBeMasked);
net.clearLayerMaskArrays();
for (int i = 0; i < minibatch; i++) {
int numSteps = height - i;
INDArray subset = inToBeMasked.get(NDArrayIndex.interval(i, i, true), NDArrayIndex.all(),
NDArrayIndex.interval(0, numSteps), NDArrayIndex.all());
assertArrayEquals(new long[] {1, depthIn, height - i, width}, subset.shape());
INDArray outSubset = net.output(subset);
INDArray outMaskedSubset = outMasked.getRow(i, true);
assertEquals("minibatch: " + i, outSubset, outMaskedSubset);
}
}
}
