OV - - Loop 147

	Loop Id: 147	Module: exec	Source: advec_cell.cpp:118-125 [...]	Coverage: 2.43%

0x2340f0 MOV	%R9,%RAX

0x2340f3 CQTO

0x2340f5 IDIV	%R15

0x2340f8 ADD	%R10D,%EDI

0x2340fb ADD	%R10D,%EAX

0x2340fe ADD	%R11D,%R8D

0x234101 ADD	%R11D,%EDX

0x234104 MOVSXD	%R8D,%R8

0x234107 MOVSXD	%EDX,%RDX

0x23410a MOVSXD	%EDI,%R13

0x23410d MOVSXD	%EAX,%R9

0x234110 MOV	-0x58(%RBP),%RDI    [18]

0x234114 MOV	%RDI,%RAX

0x234117 IMUL	%R13,%RAX

0x23411b MOV	-0xa0(%RBP),%R10    [18]

0x234122 MOV	%R10,%R11

0x234125 IMUL	%R13,%R11

0x234129 IMUL	%R9,%R10

0x23412d ADD	%R8,%R11

0x234130 ADD	%RDX,%R10

0x234133 MOV	-0x98(%RBP),%R15    [18]

0x23413a VMOVSD	(%R15,%R11,8),%XMM0    [9]

0x234140 VMOVHPD	(%R15,%R10,8),%XMM0,%XMM0    [17]

0x234146 IMUL	%R9,%RDI

0x23414a MOV	-0x90(%RBP),%R11    [18]

0x234151 MOV	%R11,%R10

0x234154 IMUL	%R13,%R10

0x234158 IMUL	%R9,%R11

0x23415c LEA	(%R10,%R8,1),%R15

0x234160 MOV	-0x88(%RBP),%R14    [18]

0x234167 VMOVSD	(%R14,%R15,8),%XMM1    [14]

0x23416d LEA	(%R11,%RDX,1),%R15

0x234171 VMOVHPD	(%R14,%R15,8),%XMM1,%XMM1    [1]

0x234177 LEA	0x1(%R10,%R8,1),%R10

0x23417c LEA	0x1(%R11,%RDX,1),%R11

0x234181 VMOVSD	(%R14,%R10,8),%XMM2    [12]

0x234187 VMOVHPD	(%R14,%R11,8),%XMM2,%XMM2    [2]

0x23418d MOV	-0x78(%RBP),%R15    [18]

0x234191 MOV	%R15,%R10

0x234194 IMUL	%R13,%R10

0x234198 IMUL	%R9,%R15

0x23419c LEA	(%R10,%R8,1),%R11

0x2341a0 MOV	-0x70(%RBP),%RBX    [18]

0x2341a4 VMOVSD	(%RBX,%R11,8),%XMM3    [4]

0x2341aa LEA	(%R15,%RDX,1),%R11

0x2341ae VMOVHPD	(%RBX,%R11,8),%XMM3,%XMM3    [10]

0x2341b4 MOV	-0xc8(%RBP),%R14    [18]

0x2341bb MOV	%R14,%R11

0x2341be IMUL	%R13,%R11

0x2341c2 ADD	%R8,%R10

0x2341c5 INC	%R10

0x2341c8 VMOVSD	(%RBX,%R10,8),%XMM4    [15]

0x2341ce MOV	%R14,%R10

0x2341d1 ADD	%RDX,%R15

0x2341d4 INC	%R15

0x2341d7 VMOVHPD	(%RBX,%R15,8),%XMM4,%XMM4    [8]

0x2341dd IMUL	%R9,%R10

0x2341e1 MOV	-0x68(%RBP),%RBX    [18]

0x2341e5 IMUL	%RBX,%R13

0x2341e9 IMUL	%RBX,%R9

0x2341ed LEA	(%R8,%R13,1),%R15

0x2341f1 MOV	-0x60(%RBP),%RBX    [18]

0x2341f5 VMOVSD	(%RBX,%R15,8),%XMM5    [3]

0x2341fb LEA	(%R9,%RDX,1),%R15

0x2341ff VMOVHPD	(%RBX,%R15,8),%XMM5,%XMM5    [6]

0x234205 ADD	%R8,%RAX

0x234208 ADD	%R8,%R11

0x23420b ADD	%R13,%R8

0x23420e INC	%R8

0x234211 VMOVSD	(%RBX,%R8,8),%XMM6    [11]

0x234217 LEA	(%R9,%RDX,1),%R8

0x23421b INC	%R8

0x23421e VMOVHPD	(%RBX,%R8,8),%XMM6,%XMM6    [19]

0x234224 ADD	%RDX,%RDI

0x234227 MOV	-0x50(%RBP),%R8    [18]

0x23422b VMOVSD	(%R8,%RAX,8),%XMM7    [5]

0x234231 VMOVHPD	(%R8,%RDI,8),%XMM7,%XMM7    [7]

0x234237 ADD	%RDX,%R10

0x23423a VMULPD	%XMM0,%XMM7,%XMM7

0x23423e VADDPD	%XMM1,%XMM7,%XMM1

0x234242 VSUBPD	%XMM2,%XMM1,%XMM1

0x234246 VADDPD	%XMM5,%XMM0,%XMM0

0x23424a MOV	-0x80(%RBP),%RDX    [18]

0x23424e VMOVSD	(%RDX,%R11,8),%XMM2    [16]

0x234254 VMOVHPD	(%RDX,%R10,8),%XMM2,%XMM2    [13]

0x23425a VSUBPD	%XMM6,%XMM0,%XMM0

0x23425e VDIVPD	%XMM0,%XMM1,%XMM0

0x234262 VMOVLPD	%XMM0,(%R8,%RAX,8)    [5]

0x234268 VMOVHPD	%XMM0,(%R8,%RDI,8)    [7]

0x23426e VMULPD	%XMM7,%XMM2,%XMM0

0x234272 VADDPD	%XMM3,%XMM0,%XMM0

0x234276 VSUBPD	%XMM4,%XMM0,%XMM0

0x23427a VDIVPD	%XMM1,%XMM0,%XMM0

0x23427e VMOVLPD	%XMM0,(%RDX,%R11,8)    [16]

0x234284 VMOVHPD	%XMM0,(%RDX,%R10,8)    [13]

0x23428a ADD	$0x2,%RSI

0x23428e ADD	$-0x2,%R12

0x234292 JE	2342ee

0x234294 LEA	0x1(%RSI),%R9

0x234298 MOV	%RSI,%RAX

0x23429b SHR	$0x20,%RAX

0x23429f MOV	-0x38(%RBP),%R15    [18]

0x2342a3 JE	2342c0

0x2342a5 MOV	%RSI,%RAX

0x2342a8 CQTO

0x2342aa IDIV	%R15

0x2342ad MOV	%RDX,%R8

0x2342b0 MOV	%RAX,%RDI

0x2342b3 JMP	2342cc

0x2342c0 MOV	%ESI,%EAX

0x2342c2 XOR	%EDX,%EDX

0x2342c4 DIV	%R15D

0x2342c7 MOV	%EDX,%R8D

0x2342ca MOV	%EAX,%EDI

0x2342cc MOV	-0x30(%RBP),%R10D    [18]

0x2342d0 MOV	-0x2c(%RBP),%R11D    [18]

0x2342d4 MOV	%R9,%RAX

0x2342d7 SHR	$0x20,%RAX

0x2342db JNE	2340f0

0x2342e1 MOV	%R9D,%EAX

0x2342e4 XOR	%EDX,%EDX

0x2342e6 DIV	%R15D

0x2342e9 JMP	2340f8

/beegfs/hackathon/users/eoseret/qaas_runs/170-854-8685/intel/CloverLeafCXX/build/CloverLeafCXX/src/omp/advec_cell.cpp: 118 - 125

--------------------------------------------------------------------------------

118:     for (int j = (y_min + 1); j < (y_max + 2); j++) {

119:       for (int i = (x_min + 1); i < (x_max + 2); i++) {

120:         double pre_mass_s = density1(i, j) * pre_vol(i, j);

121:         double post_mass_s = pre_mass_s + mass_flux_x(i, j) - mass_flux_x(i + 1, j + 0);

122:         double post_ener_s = (energy1(i, j) * pre_mass_s + ener_flux(i, j) - ener_flux(i + 1, j + 0)) / post_mass_s;

123:         double advec_vol_s = pre_vol(i, j) + vol_flux_x(i, j) - vol_flux_x(i + 1, j + 0);

124:         density1(i, j) = post_mass_s / advec_vol_s;

125:         energy1(i, j) = post_ener_s;

/beegfs/hackathon/users/eoseret/qaas_runs/170-854-8685/intel/CloverLeafCXX/build/CloverLeafCXX/src/omp/context.h: 69 - 69

--------------------------------------------------------------------------------

69:   T &operator()(size_t i, size_t j) const { return data[i + j * sizeX]; }

Path /

Metric	Value
CQA speedup if no scalar integer	2.30 - 2.80
CQA speedup if FP arith vectorized	1.22
CQA speedup if fully vectorized	3.07 - 3.20
CQA speedup if no inter-iteration dependency	NA
CQA speedup if next bottleneck killed	1.16 - 1.41
Bottlenecks
Function	.omp_outlined..6
Source	advec_cell.cpp:118-125,context.h:69-69
Source loop unroll info	unrolled by 2
Source loop unroll confidence level	max
Unroll/vectorization loop type	main
Unroll factor	2
CQA cycles	23.00 - 28.00
CQA cycles if no scalar integer	10.00
CQA cycles if FP arith vectorized	18.00 - 23.00
CQA cycles if fully vectorized	7.50 - 8.75
Front-end cycles	19.83
DIV/SQRT cycles	12.81
P0 cycles	12.81
P1 cycles	12.75
P2 cycles	12.75
P3 cycles	3.88
P4 cycles	13.00
P5 cycles	13.00
P6 cycles	13.00
P7 cycles	2.00
P8 cycles	6.33
P9 cycles	6.33
P10 cycles	6.33
P11 cycles	2.00
P12 cycles	2.00
P13 cycles	23.00 - 28.00
Inter-iter dependencies cycles	1
FE+BE cycles (UFS)	NA
Stall cycles (UFS)	NA
Nb insns	114.00
Nb uops	119.00
Nb loads	33.00
Nb stores	4.00
Nb stack references	15.00
FLOP/cycle	0.87 - 0.71
Nb FLOP add-sub	12.00
Nb FLOP mul	4.00
Nb FLOP fma	0.00
Nb FLOP div	4.00
Nb FLOP rcp	0.00
Nb FLOP sqrt	0.00
Nb FLOP rsqrt	0.00
Bytes/cycle	10.53 - 12.53
Bytes prefetched	0.00
Bytes loaded	256.00
Bytes stored	32.00
Stride 0	1.00
Stride 1	0.00
Stride n	0.00
Stride unknown	13.00
Stride indirect	0.00
Vectorization ratio all	28.58
Vectorization ratio load	0.00
Vectorization ratio store	0.00
Vectorization ratio mul	100.00
Vectorization ratio add_sub	100.00
Vectorization ratio fma	NA
Vectorization ratio div_sqrt	50.00
Vectorization ratio other	0.00
Vector-efficiency ratio all	15.89
Vector-efficiency ratio load	12.50
Vector-efficiency ratio store	12.50
Vector-efficiency ratio mul	25.00
Vector-efficiency ratio add_sub	25.00
Vector-efficiency ratio fma	NA
Vector-efficiency ratio div_sqrt	17.19
Vector-efficiency ratio other	12.50

Metric	Value
CQA speedup if no scalar integer	2.30 - 2.80
CQA speedup if FP arith vectorized	1.22
CQA speedup if fully vectorized	3.07 - 3.20
CQA speedup if no inter-iteration dependency	NA
CQA speedup if next bottleneck killed	1.15 - 1.40
Bottlenecks	P8, P9,
Function	.omp_outlined..6
Source	advec_cell.cpp:118-125,context.h:69-69
Source loop unroll info	unrolled by 2
Source loop unroll confidence level	max
Unroll/vectorization loop type	main
Unroll factor	2
CQA cycles	23.00 - 28.00
CQA cycles if no scalar integer	10.00
CQA cycles if FP arith vectorized	18.00 - 23.00
CQA cycles if fully vectorized	7.50 - 8.75
Front-end cycles	20.00
DIV/SQRT cycles	13.00
P0 cycles	13.00
P1 cycles	12.75
P2 cycles	12.75
P3 cycles	4.50
P4 cycles	13.00
P5 cycles	13.00
P6 cycles	13.00
P7 cycles	2.00
P8 cycles	6.33
P9 cycles	6.33
P10 cycles	6.33
P11 cycles	2.00
P12 cycles	2.00
P13 cycles	23.00 - 28.00
Inter-iter dependencies cycles	1
FE+BE cycles (UFS)	NA
Stall cycles (UFS)	NA
Nb insns	115.00
Nb uops	120.00
Nb loads	33.00
Nb stores	4.00
Nb stack references	15.00
FLOP/cycle	0.87 - 0.71
Nb FLOP add-sub	12.00
Nb FLOP mul	4.00
Nb FLOP fma	0.00
Nb FLOP div	4.00
Nb FLOP rcp	0.00
Nb FLOP sqrt	0.00
Nb FLOP rsqrt	0.00
Bytes/cycle	10.29 - 12.52
Bytes prefetched	0.00
Bytes loaded	256.00
Bytes stored	32.00
Stride 0	1.00
Stride 1	0.00
Stride n	0.00
Stride unknown	13.00
Stride indirect	0.00
Vectorization ratio all	28.57
Vectorization ratio load	0.00
Vectorization ratio store	0.00
Vectorization ratio mul	100.00
Vectorization ratio add_sub	100.00
Vectorization ratio fma	NA
Vectorization ratio div_sqrt	50.00
Vectorization ratio other	0.00
Vector-efficiency ratio all	15.89
Vector-efficiency ratio load	12.50
Vector-efficiency ratio store	12.50
Vector-efficiency ratio mul	25.00
Vector-efficiency ratio add_sub	25.00
Vector-efficiency ratio fma	NA
Vector-efficiency ratio div_sqrt	17.19
Vector-efficiency ratio other	12.50

Metric	Value
CQA speedup if no scalar integer	2.40 - 3.40
CQA speedup if FP arith vectorized	1.17
CQA speedup if fully vectorized	2.82 - 3.09
CQA speedup if no inter-iteration dependency	NA
CQA speedup if next bottleneck killed	1.22 - 1.73
Bottlenecks	P8, P9,
Function	.omp_outlined..6
Source	advec_cell.cpp:118-125,context.h:69-69
Source loop unroll info	unrolled by 2
Source loop unroll confidence level	max
Unroll/vectorization loop type	main
Unroll factor	2
CQA cycles	24.00 - 34.00
CQA cycles if no scalar integer	10.00
CQA cycles if FP arith vectorized	19.00 - 29.00
CQA cycles if fully vectorized	8.50 - 11.00
Front-end cycles	19.67
DIV/SQRT cycles	12.75
P0 cycles	12.75
P1 cycles	12.75
P2 cycles	12.75
P3 cycles	4.00
P4 cycles	13.00
P5 cycles	13.00
P6 cycles	13.00
P7 cycles	2.00
P8 cycles	6.33
P9 cycles	6.33
P10 cycles	6.33
P11 cycles	2.00
P12 cycles	2.00
P13 cycles	24.00 - 34.00
Inter-iter dependencies cycles	1
FE+BE cycles (UFS)	NA
Stall cycles (UFS)	NA
Nb insns	114.00
Nb uops	118.00
Nb loads	33.00
Nb stores	4.00
Nb stack references	15.00
FLOP/cycle	0.83 - 0.59
Nb FLOP add-sub	12.00
Nb FLOP mul	4.00
Nb FLOP fma	0.00
Nb FLOP div	4.00
Nb FLOP rcp	0.00
Nb FLOP sqrt	0.00
Nb FLOP rsqrt	0.00
Bytes/cycle	8.47 - 12.00
Bytes prefetched	0.00
Bytes loaded	256.00
Bytes stored	32.00
Stride 0	1.00
Stride 1	0.00
Stride n	0.00
Stride unknown	13.00
Stride indirect	0.00
Vectorization ratio all	27.78
Vectorization ratio load	0.00
Vectorization ratio store	0.00
Vectorization ratio mul	100.00
Vectorization ratio add_sub	100.00
Vectorization ratio fma	NA
Vectorization ratio div_sqrt	50.00
Vectorization ratio other	0.00
Vector-efficiency ratio all	15.97
Vector-efficiency ratio load	12.50
Vector-efficiency ratio store	12.50
Vector-efficiency ratio mul	25.00
Vector-efficiency ratio add_sub	25.00
Vector-efficiency ratio fma	NA
Vector-efficiency ratio div_sqrt	18.75
Vector-efficiency ratio other	12.50

Metric	Value
CQA speedup if no scalar integer	2.20
CQA speedup if FP arith vectorized	1.29
CQA speedup if fully vectorized	3.38
CQA speedup if no inter-iteration dependency	NA
CQA speedup if next bottleneck killed	1.10
Bottlenecks	P8, P9,
Function	.omp_outlined..6
Source	advec_cell.cpp:118-125,context.h:69-69
Source loop unroll info	unrolled by 2
Source loop unroll confidence level	max
Unroll/vectorization loop type	main
Unroll factor	2
CQA cycles	22.00
CQA cycles if no scalar integer	10.00
CQA cycles if FP arith vectorized	17.00
CQA cycles if fully vectorized	6.50
Front-end cycles	20.00
DIV/SQRT cycles	12.75
P0 cycles	12.75
P1 cycles	12.75
P2 cycles	12.75
P3 cycles	4.00
P4 cycles	13.00
P5 cycles	13.00
P6 cycles	13.00
P7 cycles	2.00
P8 cycles	6.33
P9 cycles	6.33
P10 cycles	6.33
P11 cycles	2.00
P12 cycles	2.00
P13 cycles	22.00
Inter-iter dependencies cycles	1
FE+BE cycles (UFS)	NA
Stall cycles (UFS)	NA
Nb insns	114.00
Nb uops	120.00
Nb loads	33.00
Nb stores	4.00
Nb stack references	15.00
FLOP/cycle	0.91
Nb FLOP add-sub	12.00
Nb FLOP mul	4.00
Nb FLOP fma	0.00
Nb FLOP div	4.00
Nb FLOP rcp	0.00
Nb FLOP sqrt	0.00
Nb FLOP rsqrt	0.00
Bytes/cycle	13.09
Bytes prefetched	0.00
Bytes loaded	256.00
Bytes stored	32.00
Stride 0	1.00
Stride 1	0.00
Stride n	0.00
Stride unknown	13.00
Stride indirect	0.00
Vectorization ratio all	29.41
Vectorization ratio load	0.00
Vectorization ratio store	0.00
Vectorization ratio mul	100.00
Vectorization ratio add_sub	100.00
Vectorization ratio fma	NA
Vectorization ratio div_sqrt	50.00
Vectorization ratio other	NA
Vector-efficiency ratio all	15.81
Vector-efficiency ratio load	12.50
Vector-efficiency ratio store	12.50
Vector-efficiency ratio mul	25.00
Vector-efficiency ratio add_sub	25.00
Vector-efficiency ratio fma	NA
Vector-efficiency ratio div_sqrt	15.63
Vector-efficiency ratio other	NA

Metric	Value
CQA speedup if no scalar integer	2.30 - 2.80
CQA speedup if FP arith vectorized	1.22
CQA speedup if fully vectorized	3.07 - 3.20
CQA speedup if no inter-iteration dependency	NA
CQA speedup if next bottleneck killed	1.17 - 1.42
Bottlenecks	P8, P9,
Function	.omp_outlined..6
Source	advec_cell.cpp:118-125,context.h:69-69
Source loop unroll info	unrolled by 2
Source loop unroll confidence level	max
Unroll/vectorization loop type	main
Unroll factor	2
CQA cycles	23.00 - 28.00
CQA cycles if no scalar integer	10.00
CQA cycles if FP arith vectorized	18.00 - 23.00
CQA cycles if fully vectorized	7.50 - 8.75
Front-end cycles	19.67
DIV/SQRT cycles	12.75
P0 cycles	12.75
P1 cycles	12.75
P2 cycles	12.75
P3 cycles	3.00
P4 cycles	13.00
P5 cycles	13.00
P6 cycles	13.00
P7 cycles	2.00
P8 cycles	6.33
P9 cycles	6.33
P10 cycles	6.33
P11 cycles	2.00
P12 cycles	2.00
P13 cycles	23.00 - 28.00
Inter-iter dependencies cycles	1
FE+BE cycles (UFS)	NA
Stall cycles (UFS)	NA
Nb insns	113.00
Nb uops	118.00
Nb loads	33.00
Nb stores	4.00
Nb stack references	15.00
FLOP/cycle	0.87 - 0.71
Nb FLOP add-sub	12.00
Nb FLOP mul	4.00
Nb FLOP fma	0.00
Nb FLOP div	4.00
Nb FLOP rcp	0.00
Nb FLOP sqrt	0.00
Nb FLOP rsqrt	0.00
Bytes/cycle	10.29 - 12.52
Bytes prefetched	0.00
Bytes loaded	256.00
Bytes stored	32.00
Stride 0	1.00
Stride 1	0.00
Stride n	0.00
Stride unknown	13.00
Stride indirect	0.00
Vectorization ratio all	28.57
Vectorization ratio load	0.00
Vectorization ratio store	0.00
Vectorization ratio mul	100.00
Vectorization ratio add_sub	100.00
Vectorization ratio fma	NA
Vectorization ratio div_sqrt	50.00
Vectorization ratio other	0.00
Vector-efficiency ratio all	15.89
Vector-efficiency ratio load	12.50
Vector-efficiency ratio store	12.50
Vector-efficiency ratio mul	25.00
Vector-efficiency ratio add_sub	25.00
Vector-efficiency ratio fma	NA
Vector-efficiency ratio div_sqrt	17.19
Vector-efficiency ratio other	12.50

Path /

Average path: Display a virtual path defined by average values of all real paths

Function	.omp_outlined..6
Source file and lines	advec_cell.cpp:118-125
Module	exec

The loop is defined in:

/beegfs/hackathon/users/eoseret/qaas_runs/170-854-8685/intel/CloverLeafCXX/build/CloverLeafCXX/src/omp/advec_cell.cpp:118-125
/beegfs/hackathon/users/eoseret/qaas_runs/170-854-8685/intel/CloverLeafCXX/build/CloverLeafCXX/src/omp/context.h:69

It is main loop of related source loop which is unrolled by 2 (including vectorization).
This loop has 4 execution paths.

The presence of multiple execution paths is typically the main/first bottleneck.
Try to simplify control inside loop: ideally, try to remove all conditional expressions, for example by (if applicable):

hoisting them (moving them outside the loop)
turning them into conditional moves, MIN or MAX

Ex: if (x<0) x=0 => x = (x<0 ? 0 : x) (or MAX(0,x) after defining the corresponding macro)

gain
potential
hint
expert

Vectorization

Your loop is poorly vectorized. Only 15% of vector register length is used (average across all SSE/AVX instructions). By fully vectorizing your loop, you can lower the cost of an iteration from 28.00 to 8.75 cycles (3.20x speedup).

Details

28% of SSE/AVX instructions are used in vector version (process two or more data elements in vector registers):

0% of SSE/AVX loads are used in vector version.
0% of SSE/AVX stores are used in vector version.
50% of SSE/AVX divide and square root instructions are used in vector version.
0% of SSE/AVX instructions that are not load, store, addition, subtraction nor multiply instructions are used in vector version.

Since your execution units are vector units, only a fully vectorized loop can use their full power.

Workaround

Try another compiler or update/tune your current one:
- recompile with ffast-math (included in Ofast) to extend loop vectorization to FP reductions.
Remove inter-iterations dependences from your loop and make it unit-stride:
- If your arrays have 2 or more dimensions, check whether elements are accessed contiguously and, otherwise, try to permute loops accordingly: C storage order is row-major: for(i) for(j) a[j][i] = b[j][i]; (slow, non stride 1) => for(i) for(j) a[i][j] = b[i][j]; (fast, stride 1)
- If your loop streams arrays of structures (AoS), try to use structures of arrays instead (SoA): for(i) a[i].x = b[i].x; (slow, non stride 1) => for(i) a.x[i] = b.x[i]; (fast, stride 1)

Expensive FP math instructions/calls

Detected performance impact from expensive FP math instructions/calls. By removing/reexpressing them, you can lower the cost of an iteration from 28.00 to 19.50 cycles (1.44x speedup).

FMA

Presence of both ADD/SUB and MUL operations.

Workaround

Pass to your compiler a micro-architecture specialization option:
- use march=native
Try to change order in which elements are evaluated (using parentheses) in arithmetic expressions containing both ADD/SUB and MUL operations to enable your compiler to generate FMA instructions wherever possible. For instance a + b*c is a valid FMA (MUL then ADD). However (a+b)* c cannot be translated into an FMA (ADD then MUL).

Complex instructions

Detected COMPLEX INSTRUCTIONS.

Details

These instructions generate more than one micro-operation and only one of them can be decoded during a cycle and the extra micro-operations increase pressure on execution units.

DIV: 1.33 occurrences
IDIV: 1.33 occurrences
LEA: 2 occurrences
VMOVHPD: 2 occurrences

Conversion instructions

Detected expensive conversion instructions.

Details

CQTO: 1.33 occurrences

Workaround

Avoid mixing data with different types. In particular, check if the type of constants is the same as array elements.

Type of elements and instruction set

10 SSE or AVX instructions are processing arithmetic or math operations on double precision FP elements in vector mode (two at a time).

Matching between your loop (in the source code) and the binary loop

The binary loop is composed of 20 FP arithmetical operations:

12: addition or subtraction
4: multiply
4: divide

The binary loop is loading 256 bytes (32 double precision FP elements). The binary loop is storing 32 bytes (4 double precision FP elements).

Arithmetic intensity

Arithmetic intensity is 0.07 FP operations per loaded or stored byte.

General properties

nb instructions	114
nb uops	119
loop length	474.50
used x86 registers	14
used mmx registers	0
used xmm registers	8
used ymm registers	0
used zmm registers	0
nb stack references	15
ADD-SUB / MUL ratio	3.00

Front-end

MACRO FUSION NOT POSSIBLE

micro-operation queue	19.83 cycles
front end	19.83 cycles

Back-end

	ALU0/BRU0	ALU1	ALU2	ALU3	BRU1	AGU0	AGU1	AGU2	FP0	FP1	FP2	FP3	FP4	FP5
uops	12.81	12.81	12.75	12.75	3.88	13.00	13.00	13.00	2.00	6.33	6.33	6.33	2.00	2.00
cycles	12.81	12.81	12.75	12.75	3.88	13.00	13.00	13.00	2.00	6.33	6.33	6.33	2.00	2.00

Execution ports to units layout:

ALU0/BRU0: ALU
ALU1: ALU
ALU2: ALU
ALU3: ALU
BRU1:
AGU0 (256 bits): store address, load
AGU1 (256 bits): store address, load
AGU2 (256 bits): store address, load
FP0 (256 bits): VPU, DIV/SQRT
FP1 (256 bits): VPU, DIV/SQRT
FP2 (256 bits): VPU
FP3 (256 bits): VPU
FP4 (256 bits): FP store data
FP5 (256 bits): FP store data

Cycles executing div or sqrt instructions	23.00-28.00
Longest recurrence chain latency (RecMII)	1.00

Cycles summary

Front-end	19.83
Dispatch	13.00
DIV/SQRT	23.00-28.00
Data deps.	1.00
Overall L1	23.00-28.00

Vectorization ratios

INT

all	0%
load	NA (no load vectorizable/vectorized instructions)
store	NA (no store vectorizable/vectorized instructions)
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	NA (no add-sub vectorizable/vectorized instructions)
fma	NA (no fma vectorizable/vectorized instructions)
other	0%

all	31%
load	0%
store	0%
mul	100%
add-sub	100%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	100%
other	NA (no other vectorizable/vectorized instructions)

INT+FP

all	28%
load	0%
store	0%
mul	100%
add-sub	100%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	50%
other	0%

Vector efficiency ratios

INT

all	9%
load	NA (no load vectorizable/vectorized instructions)
store	NA (no store vectorizable/vectorized instructions)
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	NA (no add-sub vectorizable/vectorized instructions)
fma	NA (no fma vectorizable/vectorized instructions)
other	9%

all	16%
load	12%
store	12%
mul	25%
add-sub	25%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	25%
other	NA (no other vectorizable/vectorized instructions)

INT+FP

all	15%
load	12%
store	12%
mul	25%
add-sub	25%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	17%
other	12%

Cycles and memory resources usage

Assuming all data fit into the L1 cache, each iteration of the binary loop takes 28.00 cycles. At this rate:

9% of peak load performance is reached (9.36 out of 96.00 bytes loaded per cycle (GB/s @ 1GHz))
1% of peak store performance is reached (1.17 out of 64.00 bytes stored per cycle (GB/s @ 1GHz))

Function	.omp_outlined..6
Source file and lines	advec_cell.cpp:118-125
Module	exec

The loop is defined in:

/beegfs/hackathon/users/eoseret/qaas_runs/170-854-8685/intel/CloverLeafCXX/build/CloverLeafCXX/src/omp/advec_cell.cpp:118-125
/beegfs/hackathon/users/eoseret/qaas_runs/170-854-8685/intel/CloverLeafCXX/build/CloverLeafCXX/src/omp/context.h:69

hoisting them (moving them outside the loop)
turning them into conditional moves, MIN or MAX

Ex: if (x<0) x=0 => x = (x<0 ? 0 : x) (or MAX(0,x) after defining the corresponding macro)

gain
potential
hint
expert

Code clean check

Detected a slowdown caused by scalar integer instructions (typically used for address computation). By removing them, you can lower the cost of an iteration from 28.00 to 10.00 cycles (2.80x speedup).

Workaround

Try to reorganize arrays of structures to structures of arrays
Consider to permute loops (see vectorization gain report)

Vectorization

Details

28% of SSE/AVX instructions are used in vector version (process two or more data elements in vector registers):

0% of SSE/AVX loads are used in vector version.
0% of SSE/AVX stores are used in vector version.
50% of SSE/AVX divide and square root instructions are used in vector version.
0% of SSE/AVX instructions that are not load, store, addition, subtraction nor multiply instructions are used in vector version.

Since your execution units are vector units, only a fully vectorized loop can use their full power.

Workaround

Try another compiler or update/tune your current one:
- recompile with ffast-math (included in Ofast) to extend loop vectorization to FP reductions.
Remove inter-iterations dependences from your loop and make it unit-stride:
- If your arrays have 2 or more dimensions, check whether elements are accessed contiguously and, otherwise, try to permute loops accordingly: C storage order is row-major: for(i) for(j) a[j][i] = b[j][i]; (slow, non stride 1) => for(i) for(j) a[i][j] = b[i][j]; (fast, stride 1)
- If your loop streams arrays of structures (AoS), try to use structures of arrays instead (SoA): for(i) a[i].x = b[i].x; (slow, non stride 1) => for(i) a.x[i] = b.x[i]; (fast, stride 1)

Execution units bottlenecks

Performance is limited by execution of divide and square root operations (the divide/square root unit is a bottleneck). By removing all these bottlenecks, you can lower the cost of an iteration from 28.00 to 20.00 cycles (1.40x speedup).

Workaround

Reduce the number of division or square root instructions:
- If denominator is constant over iterations, use reciprocal (replace x/y with x*(1/y)). Check precision impact. This will be done by your compiler with ffast-math or Ofast
Check whether you really need double precision. If not, switch to single precision to speedup execution

Expensive FP math instructions/calls

Detected performance impact from expensive FP math instructions/calls. By removing/reexpressing them, you can lower the cost of an iteration from 28.00 to 19.67 cycles (1.42x speedup).

FMA

Presence of both ADD/SUB and MUL operations.

Workaround

Pass to your compiler a micro-architecture specialization option:
- use march=native
Try to change order in which elements are evaluated (using parentheses) in arithmetic expressions containing both ADD/SUB and MUL operations to enable your compiler to generate FMA instructions wherever possible. For instance a + b*c is a valid FMA (MUL then ADD). However (a+b)* c cannot be translated into an FMA (ADD then MUL).

Complex instructions

Detected COMPLEX INSTRUCTIONS.

Details

These instructions generate more than one micro-operation and only one of them can be decoded during a cycle and the extra micro-operations increase pressure on execution units.

DIV: 1 occurrences
IDIV: 1 occurrences
LEA: 2 occurrences
VMOVHPD: 2 occurrences

Slow data structures access

Detected data structures (typically arrays) that cannot be efficiently read/written

Details

Constant unknown stride: 13 occurrence(s)

Non-unit stride (uncontiguous) accesses are not efficiently using data caches

Workaround

Try to reorganize arrays of structures to structures of arrays
Consider to permute loops (see vectorization gain report)

Conversion instructions

Detected expensive conversion instructions.

Details

CQTO: 1 occurrences

Workaround

Avoid mixing data with different types. In particular, check if the type of constants is the same as array elements.

Type of elements and instruction set

10 SSE or AVX instructions are processing arithmetic or math operations on double precision FP elements in vector mode (two at a time).

Matching between your loop (in the source code) and the binary loop

The binary loop is composed of 20 FP arithmetical operations:

12: addition or subtraction
4: multiply
4: divide

The binary loop is loading 256 bytes (32 double precision FP elements). The binary loop is storing 32 bytes (4 double precision FP elements).

Arithmetic intensity

Arithmetic intensity is 0.07 FP operations per loaded or stored byte.

General properties

nb instructions	115
nb uops	120
loop length	479
used x86 registers	14
used mmx registers	0
used xmm registers	8
used ymm registers	0
used zmm registers	0
nb stack references	15
ADD-SUB / MUL ratio	3.00

Front-end

MACRO FUSION NOT POSSIBLE FIT IN UOP CACHE

micro-operation queue	20.00 cycles
front end	20.00 cycles

Back-end

	ALU0/BRU0	ALU1	ALU2	ALU3	BRU1	AGU0	AGU1	AGU2	FP0	FP1	FP2	FP3	FP4	FP5
uops	13.00	13.00	12.75	12.75	4.50	13.00	13.00	13.00	2.00	6.33	6.33	6.33	2.00	2.00
cycles	13.00	13.00	12.75	12.75	4.50	13.00	13.00	13.00	2.00	6.33	6.33	6.33	2.00	2.00

Execution ports to units layout:

ALU0/BRU0: ALU
ALU1: ALU
ALU2: ALU
ALU3: ALU
BRU1:
AGU0 (256 bits): store address, load
AGU1 (256 bits): store address, load
AGU2 (256 bits): store address, load
FP0 (256 bits): VPU, DIV/SQRT
FP1 (256 bits): VPU, DIV/SQRT
FP2 (256 bits): VPU
FP3 (256 bits): VPU
FP4 (256 bits): FP store data
FP5 (256 bits): FP store data

Cycles executing div or sqrt instructions	23.00-28.00
Longest recurrence chain latency (RecMII)	1.00

Cycles summary

Front-end	20.00
Dispatch	13.00
DIV/SQRT	23.00-28.00
Data deps.	1.00
Overall L1	23.00-28.00

Vectorization ratios

INT

all	0%
load	NA (no load vectorizable/vectorized instructions)
store	NA (no store vectorizable/vectorized instructions)
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	NA (no add-sub vectorizable/vectorized instructions)
fma	NA (no fma vectorizable/vectorized instructions)
other	0%

all	31%
load	0%
store	0%
mul	100%
add-sub	100%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	100%
other	NA (no other vectorizable/vectorized instructions)

INT+FP

all	28%
load	0%
store	0%
mul	100%
add-sub	100%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	50%
other	0%

Vector efficiency ratios

INT

all	10%
load	NA (no load vectorizable/vectorized instructions)
store	NA (no store vectorizable/vectorized instructions)
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	NA (no add-sub vectorizable/vectorized instructions)
fma	NA (no fma vectorizable/vectorized instructions)
other	10%

all	16%
load	12%
store	12%
mul	25%
add-sub	25%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	25%
other	NA (no other vectorizable/vectorized instructions)

INT+FP

all	15%
load	12%
store	12%
mul	25%
add-sub	25%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	17%
other	12%

Cycles and memory resources usage

Assuming all data fit into the L1 cache, each iteration of the binary loop takes 28.00 cycles. At this rate:

9% of peak load performance is reached (9.14 out of 96.00 bytes loaded per cycle (GB/s @ 1GHz))
1% of peak store performance is reached (1.14 out of 64.00 bytes stored per cycle (GB/s @ 1GHz))

Front-end bottlenecks

Found no such bottlenecks.

ASM code

In the binary file, the address of the loop is: 234294

Instruction	Nb FU	ALU0/BRU0	ALU1	ALU2	ALU3	BRU1	AGU0	AGU1	AGU2	FP0	FP1	FP2	FP3	FP4	FP5	Latency	Recip. throughput
ADD %R10D,%EDI	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD %R10D,%EAX	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD %R11D,%R8D	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD %R11D,%EDX	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOVSXD %R8D,%R8	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOVSXD %EDX,%RDX	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOVSXD %EDI,%R13	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOVSXD %EAX,%R9	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV -0x58(%RBP),%RDI	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV %RDI,%RAX	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
IMUL %R13,%RAX	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
MOV -0xa0(%RBP),%R10	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV %R10,%R11	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
IMUL %R13,%R11	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
IMUL %R9,%R10	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
ADD %R8,%R11	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD %RDX,%R10	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV -0x98(%RBP),%R15	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVSD (%R15,%R11,8),%XMM0	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
VMOVHPD (%R15,%R10,8),%XMM0,%XMM0	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
IMUL %R9,%RDI	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
MOV -0x90(%RBP),%R11	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV %R11,%R10	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
IMUL %R13,%R10	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
IMUL %R9,%R11	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
LEA (%R10,%R8,1),%R15	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV -0x88(%RBP),%R14	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVSD (%R14,%R15,8),%XMM1	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
LEA (%R11,%RDX,1),%R15	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMOVHPD (%R14,%R15,8),%XMM1,%XMM1	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
LEA 0x1(%R10,%R8,1),%R10	2	0	0	0.50	0.50	0	0.33	0.33	0.33	0	0	0	0	0	0	2	0.50
LEA 0x1(%R11,%RDX,1),%R11	2	0	0	0.50	0.50	0	0.33	0.33	0.33	0	0	0	0	0	0	2	0.50
VMOVSD (%R14,%R10,8),%XMM2	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
VMOVHPD (%R14,%R11,8),%XMM2,%XMM2	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
MOV -0x78(%RBP),%R15	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV %R15,%R10	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
IMUL %R13,%R10	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
IMUL %R9,%R15	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
LEA (%R10,%R8,1),%R11	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV -0x70(%RBP),%RBX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVSD (%RBX,%R11,8),%XMM3	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
LEA (%R15,%RDX,1),%R11	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMOVHPD (%RBX,%R11,8),%XMM3,%XMM3	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
MOV -0xc8(%RBP),%R14	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV %R14,%R11	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
IMUL %R13,%R11	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
ADD %R8,%R10	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
INC %R10	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMOVSD (%RBX,%R10,8),%XMM4	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
MOV %R14,%R10	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
ADD %RDX,%R15	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
INC %R15	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMOVHPD (%RBX,%R15,8),%XMM4,%XMM4	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
IMUL %R9,%R10	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
MOV -0x68(%RBP),%RBX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
IMUL %RBX,%R13	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
IMUL %RBX,%R9	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
LEA (%R8,%R13,1),%R15	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV -0x60(%RBP),%RBX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVSD (%RBX,%R15,8),%XMM5	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
LEA (%R9,%RDX,1),%R15	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMOVHPD (%RBX,%R15,8),%XMM5,%XMM5	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
ADD %R8,%RAX	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD %R8,%R11	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD %R13,%R8	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
INC %R8	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMOVSD (%RBX,%R8,8),%XMM6	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
LEA (%R9,%RDX,1),%R8	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
INC %R8	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMOVHPD (%RBX,%R8,8),%XMM6,%XMM6	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
ADD %RDX,%RDI	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV -0x50(%RBP),%R8	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVSD (%R8,%RAX,8),%XMM7	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
VMOVHPD (%R8,%RDI,8),%XMM7,%XMM7	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
ADD %RDX,%R10	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMULPD %XMM0,%XMM7,%XMM7	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	3	0.50
VADDPD %XMM1,%XMM7,%XMM1	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
VSUBPD %XMM2,%XMM1,%XMM1	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
VADDPD %XMM5,%XMM0,%XMM0	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
MOV -0x80(%RBP),%RDX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVSD (%RDX,%R11,8),%XMM2	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
VMOVHPD (%RDX,%R10,8),%XMM2,%XMM2	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
VSUBPD %XMM6,%XMM0,%XMM0	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
VDIVPD %XMM0,%XMM1,%XMM0	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	13	5
VMOVLPD %XMM0,(%R8,%RAX,8)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0.50	0.50	1	1
VMOVHPD %XMM0,(%R8,%RDI,8)	2	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0.50	0.50	9-11	1
VMULPD %XMM7,%XMM2,%XMM0	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	3	0.50
VADDPD %XMM3,%XMM0,%XMM0	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
VSUBPD %XMM4,%XMM0,%XMM0	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
VDIVPD %XMM1,%XMM0,%XMM0	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	13	5
VMOVLPD %XMM0,(%RDX,%R11,8)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0.50	0.50	1	1
VMOVHPD %XMM0,(%RDX,%R10,8)	2	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0.50	0.50	9-11	1
ADD $0x2,%RSI	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD $-0x2,%R12	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
JE 2342ee <.omp_outlined..6+0x38e>	1	0.50	0	0	0	0.50	0	0	0	0	0	0	0	0	0	1	0.50-1
LEA 0x1(%RSI),%R9	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV %RSI,%RAX	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
SHR $0x20,%RAX	1	0	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	1	0.50
MOV -0x38(%RBP),%R15	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
JE 2342c0 <.omp_outlined..6+0x360>	1	0.50	0	0	0	0.50	0	0	0	0	0	0	0	0	0	1	0.50-1
MOV %RSI,%RAX	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
CQTO
IDIV %R15	2	2	0	0	0	0	0	0	0	0	0	0	0	0	0	9-19	7-12
MOV %RDX,%R8	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
MOV %RAX,%RDI	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
JMP 2342cc <.omp_outlined..6+0x36c>	1	0.50	0	0	0	0.50	0	0	0	0	0	0	0	0	0	0	1
MOV -0x30(%RBP),%R10D	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV -0x2c(%RBP),%R11D	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV %R9,%RAX	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
SHR $0x20,%RAX	1	0	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	1	0.50
JNE 2340f0 <.omp_outlined..6+0x190>	1	0.50	0	0	0	0.50	0	0	0	0	0	0	0	0	0	1	0.50-1
MOV %R9D,%EAX	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
XOR %EDX,%EDX	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.25
DIV %R15D	2	2	0	0	0	0	0	0	0	0	0	0	0	0	0	10-17	6
JMP 2340f8 <.omp_outlined..6+0x198>	1	0.50	0	0	0	0.50	0	0	0	0	0	0	0	0	0	0	1

Function	.omp_outlined..6
Source file and lines	advec_cell.cpp:118-125
Module	exec

The loop is defined in:

/beegfs/hackathon/users/eoseret/qaas_runs/170-854-8685/intel/CloverLeafCXX/build/CloverLeafCXX/src/omp/advec_cell.cpp:118-125
/beegfs/hackathon/users/eoseret/qaas_runs/170-854-8685/intel/CloverLeafCXX/build/CloverLeafCXX/src/omp/context.h:69

hoisting them (moving them outside the loop)
turning them into conditional moves, MIN or MAX

Ex: if (x<0) x=0 => x = (x<0 ? 0 : x) (or MAX(0,x) after defining the corresponding macro)

gain
potential
hint
expert

Code clean check

Detected a slowdown caused by scalar integer instructions (typically used for address computation). By removing them, you can lower the cost of an iteration from 34.00 to 10.00 cycles (3.40x speedup).

Workaround

Try to reorganize arrays of structures to structures of arrays
Consider to permute loops (see vectorization gain report)

Vectorization

Details

27% of SSE/AVX instructions are used in vector version (process two or more data elements in vector registers):

0% of SSE/AVX loads are used in vector version.
0% of SSE/AVX stores are used in vector version.
50% of SSE/AVX divide and square root instructions are used in vector version.
0% of SSE/AVX instructions that are not load, store, addition, subtraction nor multiply instructions are used in vector version.

Since your execution units are vector units, only a fully vectorized loop can use their full power.

Workaround

Try another compiler or update/tune your current one:
- recompile with ffast-math (included in Ofast) to extend loop vectorization to FP reductions.
Remove inter-iterations dependences from your loop and make it unit-stride:
- If your arrays have 2 or more dimensions, check whether elements are accessed contiguously and, otherwise, try to permute loops accordingly: C storage order is row-major: for(i) for(j) a[j][i] = b[j][i]; (slow, non stride 1) => for(i) for(j) a[i][j] = b[i][j]; (fast, stride 1)
- If your loop streams arrays of structures (AoS), try to use structures of arrays instead (SoA): for(i) a[i].x = b[i].x; (slow, non stride 1) => for(i) a.x[i] = b.x[i]; (fast, stride 1)

Execution units bottlenecks

Workaround

Reduce the number of division or square root instructions:
- If denominator is constant over iterations, use reciprocal (replace x/y with x*(1/y)). Check precision impact. This will be done by your compiler with ffast-math or Ofast
Check whether you really need double precision. If not, switch to single precision to speedup execution

Expensive FP math instructions/calls

Detected performance impact from expensive FP math instructions/calls. By removing/reexpressing them, you can lower the cost of an iteration from 34.00 to 19.33 cycles (1.76x speedup).

FMA

Presence of both ADD/SUB and MUL operations.

Workaround

Pass to your compiler a micro-architecture specialization option:
- use march=native
Try to change order in which elements are evaluated (using parentheses) in arithmetic expressions containing both ADD/SUB and MUL operations to enable your compiler to generate FMA instructions wherever possible. For instance a + b*c is a valid FMA (MUL then ADD). However (a+b)* c cannot be translated into an FMA (ADD then MUL).

Complex instructions

Detected COMPLEX INSTRUCTIONS.

Details

These instructions generate more than one micro-operation and only one of them can be decoded during a cycle and the extra micro-operations increase pressure on execution units.

IDIV: 2 occurrences
LEA: 2 occurrences
VMOVHPD: 2 occurrences

Slow data structures access

Detected data structures (typically arrays) that cannot be efficiently read/written

Details

Constant unknown stride: 13 occurrence(s)

Non-unit stride (uncontiguous) accesses are not efficiently using data caches

Workaround

Try to reorganize arrays of structures to structures of arrays
Consider to permute loops (see vectorization gain report)

Conversion instructions

Detected expensive conversion instructions.

Details

CQTO: 2 occurrences

Workaround

Avoid mixing data with different types. In particular, check if the type of constants is the same as array elements.

Type of elements and instruction set

10 SSE or AVX instructions are processing arithmetic or math operations on double precision FP elements in vector mode (two at a time).

Matching between your loop (in the source code) and the binary loop

The binary loop is composed of 20 FP arithmetical operations:

12: addition or subtraction
4: multiply
4: divide

The binary loop is loading 256 bytes (32 double precision FP elements). The binary loop is storing 32 bytes (4 double precision FP elements).

Arithmetic intensity

Arithmetic intensity is 0.07 FP operations per loaded or stored byte.

General properties

nb instructions	114
nb uops	118
loop length	474
used x86 registers	14
used mmx registers	0
used xmm registers	8
used ymm registers	0
used zmm registers	0
nb stack references	15
ADD-SUB / MUL ratio	3.00

Front-end

MACRO FUSION NOT POSSIBLE FIT IN UOP CACHE

micro-operation queue	19.67 cycles
front end	19.67 cycles

Back-end

	ALU0/BRU0	ALU1	ALU2	ALU3	BRU1	AGU0	AGU1	AGU2	FP0	FP1	FP2	FP3	FP4	FP5
uops	12.75	12.75	12.75	12.75	4.00	13.00	13.00	13.00	2.00	6.33	6.33	6.33	2.00	2.00
cycles	12.75	12.75	12.75	12.75	4.00	13.00	13.00	13.00	2.00	6.33	6.33	6.33	2.00	2.00

Execution ports to units layout:

ALU0/BRU0: ALU
ALU1: ALU
ALU2: ALU
ALU3: ALU
BRU1:
AGU0 (256 bits): store address, load
AGU1 (256 bits): store address, load
AGU2 (256 bits): store address, load
FP0 (256 bits): VPU, DIV/SQRT
FP1 (256 bits): VPU, DIV/SQRT
FP2 (256 bits): VPU
FP3 (256 bits): VPU
FP4 (256 bits): FP store data
FP5 (256 bits): FP store data

Cycles executing div or sqrt instructions	24.00-34.00
Longest recurrence chain latency (RecMII)	1.00

Cycles summary

Front-end	19.67
Dispatch	13.00
DIV/SQRT	24.00-34.00
Data deps.	1.00
Overall L1	24.00-34.00

Vectorization ratios

INT

all	0%
load	NA (no load vectorizable/vectorized instructions)
store	NA (no store vectorizable/vectorized instructions)
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	NA (no add-sub vectorizable/vectorized instructions)
fma	NA (no fma vectorizable/vectorized instructions)
other	0%

all	31%
load	0%
store	0%
mul	100%
add-sub	100%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	100%
other	NA (no other vectorizable/vectorized instructions)

INT+FP

all	27%
load	0%
store	0%
mul	100%
add-sub	100%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	50%
other	0%

Vector efficiency ratios

INT

all	12%
load	NA (no load vectorizable/vectorized instructions)
store	NA (no store vectorizable/vectorized instructions)
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	NA (no add-sub vectorizable/vectorized instructions)
fma	NA (no fma vectorizable/vectorized instructions)
other	12%

all	16%
load	12%
store	12%
mul	25%
add-sub	25%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	25%
other	NA (no other vectorizable/vectorized instructions)

INT+FP

all	15%
load	12%
store	12%
mul	25%
add-sub	25%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	18%
other	12%

Cycles and memory resources usage

Assuming all data fit into the L1 cache, each iteration of the binary loop takes 34.00 cycles. At this rate:

7% of peak load performance is reached (7.53 out of 96.00 bytes loaded per cycle (GB/s @ 1GHz))
1% of peak store performance is reached (0.94 out of 64.00 bytes stored per cycle (GB/s @ 1GHz))

Front-end bottlenecks

Found no such bottlenecks.

ASM code

In the binary file, the address of the loop is: 234294

Instruction	Nb FU	ALU0/BRU0	ALU1	ALU2	ALU3	BRU1	AGU0	AGU1	AGU2	FP0	FP1	FP2	FP3	FP4	FP5	Latency	Recip. throughput
MOV %R9,%RAX	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
CQTO
IDIV %R15	2	2	0	0	0	0	0	0	0	0	0	0	0	0	0	9-19	7-12
ADD %R10D,%EDI	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD %R10D,%EAX	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD %R11D,%R8D	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD %R11D,%EDX	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOVSXD %R8D,%R8	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOVSXD %EDX,%RDX	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOVSXD %EDI,%R13	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOVSXD %EAX,%R9	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV -0x58(%RBP),%RDI	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV %RDI,%RAX	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
IMUL %R13,%RAX	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
MOV -0xa0(%RBP),%R10	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV %R10,%R11	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
IMUL %R13,%R11	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
IMUL %R9,%R10	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
ADD %R8,%R11	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD %RDX,%R10	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV -0x98(%RBP),%R15	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVSD (%R15,%R11,8),%XMM0	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
VMOVHPD (%R15,%R10,8),%XMM0,%XMM0	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
IMUL %R9,%RDI	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
MOV -0x90(%RBP),%R11	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV %R11,%R10	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
IMUL %R13,%R10	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
IMUL %R9,%R11	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
LEA (%R10,%R8,1),%R15	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV -0x88(%RBP),%R14	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVSD (%R14,%R15,8),%XMM1	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
LEA (%R11,%RDX,1),%R15	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMOVHPD (%R14,%R15,8),%XMM1,%XMM1	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
LEA 0x1(%R10,%R8,1),%R10	2	0	0	0.50	0.50	0	0.33	0.33	0.33	0	0	0	0	0	0	2	0.50
LEA 0x1(%R11,%RDX,1),%R11	2	0	0	0.50	0.50	0	0.33	0.33	0.33	0	0	0	0	0	0	2	0.50
VMOVSD (%R14,%R10,8),%XMM2	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
VMOVHPD (%R14,%R11,8),%XMM2,%XMM2	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
MOV -0x78(%RBP),%R15	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV %R15,%R10	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
IMUL %R13,%R10	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
IMUL %R9,%R15	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
LEA (%R10,%R8,1),%R11	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV -0x70(%RBP),%RBX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVSD (%RBX,%R11,8),%XMM3	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
LEA (%R15,%RDX,1),%R11	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMOVHPD (%RBX,%R11,8),%XMM3,%XMM3	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
MOV -0xc8(%RBP),%R14	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV %R14,%R11	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
IMUL %R13,%R11	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
ADD %R8,%R10	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
INC %R10	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMOVSD (%RBX,%R10,8),%XMM4	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
MOV %R14,%R10	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
ADD %RDX,%R15	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
INC %R15	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMOVHPD (%RBX,%R15,8),%XMM4,%XMM4	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
IMUL %R9,%R10	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
MOV -0x68(%RBP),%RBX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
IMUL %RBX,%R13	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
IMUL %RBX,%R9	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
LEA (%R8,%R13,1),%R15	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV -0x60(%RBP),%RBX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVSD (%RBX,%R15,8),%XMM5	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
LEA (%R9,%RDX,1),%R15	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMOVHPD (%RBX,%R15,8),%XMM5,%XMM5	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
ADD %R8,%RAX	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD %R8,%R11	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD %R13,%R8	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
INC %R8	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMOVSD (%RBX,%R8,8),%XMM6	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
LEA (%R9,%RDX,1),%R8	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
INC %R8	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMOVHPD (%RBX,%R8,8),%XMM6,%XMM6	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
ADD %RDX,%RDI	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV -0x50(%RBP),%R8	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVSD (%R8,%RAX,8),%XMM7	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
VMOVHPD (%R8,%RDI,8),%XMM7,%XMM7	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
ADD %RDX,%R10	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMULPD %XMM0,%XMM7,%XMM7	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	3	0.50
VADDPD %XMM1,%XMM7,%XMM1	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
VSUBPD %XMM2,%XMM1,%XMM1	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
VADDPD %XMM5,%XMM0,%XMM0	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
MOV -0x80(%RBP),%RDX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVSD (%RDX,%R11,8),%XMM2	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
VMOVHPD (%RDX,%R10,8),%XMM2,%XMM2	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
VSUBPD %XMM6,%XMM0,%XMM0	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
VDIVPD %XMM0,%XMM1,%XMM0	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	13	5
VMOVLPD %XMM0,(%R8,%RAX,8)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0.50	0.50	1	1
VMOVHPD %XMM0,(%R8,%RDI,8)	2	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0.50	0.50	9-11	1
VMULPD %XMM7,%XMM2,%XMM0	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	3	0.50
VADDPD %XMM3,%XMM0,%XMM0	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
VSUBPD %XMM4,%XMM0,%XMM0	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
VDIVPD %XMM1,%XMM0,%XMM0	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	13	5
VMOVLPD %XMM0,(%RDX,%R11,8)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0.50	0.50	1	1
VMOVHPD %XMM0,(%RDX,%R10,8)	2	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0.50	0.50	9-11	1
ADD $0x2,%RSI	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD $-0x2,%R12	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
JE 2342ee <.omp_outlined..6+0x38e>	1	0.50	0	0	0	0.50	0	0	0	0	0	0	0	0	0	1	0.50-1
LEA 0x1(%RSI),%R9	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV %RSI,%RAX	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
SHR $0x20,%RAX	1	0	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	1	0.50
MOV -0x38(%RBP),%R15	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
JE 2342c0 <.omp_outlined..6+0x360>	1	0.50	0	0	0	0.50	0	0	0	0	0	0	0	0	0	1	0.50-1
MOV %RSI,%RAX	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
CQTO
IDIV %R15	2	2	0	0	0	0	0	0	0	0	0	0	0	0	0	9-19	7-12
MOV %RDX,%R8	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
MOV %RAX,%RDI	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
JMP 2342cc <.omp_outlined..6+0x36c>	1	0.50	0	0	0	0.50	0	0	0	0	0	0	0	0	0	0	1
MOV -0x30(%RBP),%R10D	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV -0x2c(%RBP),%R11D	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV %R9,%RAX	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
SHR $0x20,%RAX	1	0	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	1	0.50
JNE 2340f0 <.omp_outlined..6+0x190>	1	0.50	0	0	0	0.50	0	0	0	0	0	0	0	0	0	1	0.50-1

Function	.omp_outlined..6
Source file and lines	advec_cell.cpp:118-125
Module	exec

The loop is defined in:

/beegfs/hackathon/users/eoseret/qaas_runs/170-854-8685/intel/CloverLeafCXX/build/CloverLeafCXX/src/omp/advec_cell.cpp:118-125
/beegfs/hackathon/users/eoseret/qaas_runs/170-854-8685/intel/CloverLeafCXX/build/CloverLeafCXX/src/omp/context.h:69

hoisting them (moving them outside the loop)
turning them into conditional moves, MIN or MAX

Ex: if (x<0) x=0 => x = (x<0 ? 0 : x) (or MAX(0,x) after defining the corresponding macro)

gain
potential
hint
expert

Code clean check

Detected a slowdown caused by scalar integer instructions (typically used for address computation). By removing them, you can lower the cost of an iteration from 22.00 to 10.00 cycles (2.20x speedup).

Workaround

Try to reorganize arrays of structures to structures of arrays
Consider to permute loops (see vectorization gain report)

Vectorization

Details

29% of SSE/AVX instructions are used in vector version (process two or more data elements in vector registers):

0% of SSE/AVX loads are used in vector version.
0% of SSE/AVX stores are used in vector version.
50% of SSE/AVX divide and square root instructions are used in vector version.

Since your execution units are vector units, only a fully vectorized loop can use their full power.

Workaround

Try another compiler or update/tune your current one:
- recompile with ffast-math (included in Ofast) to extend loop vectorization to FP reductions.
Remove inter-iterations dependences from your loop and make it unit-stride:
- If your arrays have 2 or more dimensions, check whether elements are accessed contiguously and, otherwise, try to permute loops accordingly: C storage order is row-major: for(i) for(j) a[j][i] = b[j][i]; (slow, non stride 1) => for(i) for(j) a[i][j] = b[i][j]; (fast, stride 1)
- If your loop streams arrays of structures (AoS), try to use structures of arrays instead (SoA): for(i) a[i].x = b[i].x; (slow, non stride 1) => for(i) a.x[i] = b.x[i]; (fast, stride 1)

Execution units bottlenecks

Workaround

Reduce the number of division or square root instructions:
- If denominator is constant over iterations, use reciprocal (replace x/y with x*(1/y)). Check precision impact. This will be done by your compiler with ffast-math or Ofast
Check whether you really need double precision. If not, switch to single precision to speedup execution

Expensive FP math instructions/calls

Detected performance impact from expensive FP math instructions/calls. By removing/reexpressing them, you can lower the cost of an iteration from 22.00 to 19.67 cycles (1.12x speedup).

FMA

Presence of both ADD/SUB and MUL operations.

Workaround

Pass to your compiler a micro-architecture specialization option:
- use march=native
Try to change order in which elements are evaluated (using parentheses) in arithmetic expressions containing both ADD/SUB and MUL operations to enable your compiler to generate FMA instructions wherever possible. For instance a + b*c is a valid FMA (MUL then ADD). However (a+b)* c cannot be translated into an FMA (ADD then MUL).

Complex instructions

Detected COMPLEX INSTRUCTIONS.

Details

These instructions generate more than one micro-operation and only one of them can be decoded during a cycle and the extra micro-operations increase pressure on execution units.

DIV: 2 occurrences
LEA: 2 occurrences
VMOVHPD: 2 occurrences

Slow data structures access

Detected data structures (typically arrays) that cannot be efficiently read/written

Details

Constant unknown stride: 13 occurrence(s)

Non-unit stride (uncontiguous) accesses are not efficiently using data caches

Workaround

Try to reorganize arrays of structures to structures of arrays
Consider to permute loops (see vectorization gain report)

Type of elements and instruction set

10 SSE or AVX instructions are processing arithmetic or math operations on double precision FP elements in vector mode (two at a time).

Matching between your loop (in the source code) and the binary loop

The binary loop is composed of 20 FP arithmetical operations:

12: addition or subtraction
4: multiply
4: divide

The binary loop is loading 256 bytes (32 double precision FP elements). The binary loop is storing 32 bytes (4 double precision FP elements).

Arithmetic intensity

Arithmetic intensity is 0.07 FP operations per loaded or stored byte.

General properties

nb instructions	114
nb uops	120
loop length	475
used x86 registers	14
used mmx registers	0
used xmm registers	8
used ymm registers	0
used zmm registers	0
nb stack references	15
ADD-SUB / MUL ratio	3.00

Front-end

MACRO FUSION NOT POSSIBLE FIT IN UOP CACHE

micro-operation queue	20.00 cycles
front end	20.00 cycles

Back-end

	ALU0/BRU0	ALU1	ALU2	ALU3	BRU1	AGU0	AGU1	AGU2	FP0	FP1	FP2	FP3	FP4	FP5
uops	12.75	12.75	12.75	12.75	4.00	13.00	13.00	13.00	2.00	6.33	6.33	6.33	2.00	2.00
cycles	12.75	12.75	12.75	12.75	4.00	13.00	13.00	13.00	2.00	6.33	6.33	6.33	2.00	2.00

Execution ports to units layout:

ALU0/BRU0: ALU
ALU1: ALU
ALU2: ALU
ALU3: ALU
BRU1:
AGU0 (256 bits): store address, load
AGU1 (256 bits): store address, load
AGU2 (256 bits): store address, load
FP0 (256 bits): VPU, DIV/SQRT
FP1 (256 bits): VPU, DIV/SQRT
FP2 (256 bits): VPU
FP3 (256 bits): VPU
FP4 (256 bits): FP store data
FP5 (256 bits): FP store data

Cycles executing div or sqrt instructions	22.00
Longest recurrence chain latency (RecMII)	1.00

Cycles summary

Front-end	20.00
Dispatch	13.00
DIV/SQRT	22.00
Data deps.	1.00
Overall L1	22.00

Vectorization ratios

INT

all	0%
load	NA (no load vectorizable/vectorized instructions)
store	NA (no store vectorizable/vectorized instructions)
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	NA (no add-sub vectorizable/vectorized instructions)
fma	NA (no fma vectorizable/vectorized instructions)
other	0%

all	31%
load	0%
store	0%
mul	100%
add-sub	100%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	100%
other	NA (no other vectorizable/vectorized instructions)

INT+FP

all	29%
load	0%
store	0%
mul	100%
add-sub	100%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	50%
other	NA (no other vectorizable/vectorized instructions)

Vector efficiency ratios

INT

all	6%
load	NA (no load vectorizable/vectorized instructions)
store	NA (no store vectorizable/vectorized instructions)
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	NA (no add-sub vectorizable/vectorized instructions)
fma	NA (no fma vectorizable/vectorized instructions)
other	6%

all	16%
load	12%
store	12%
mul	25%
add-sub	25%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	25%
other	NA (no other vectorizable/vectorized instructions)

INT+FP

all	15%
load	12%
store	12%
mul	25%
add-sub	25%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	15%
other	NA (no other vectorizable/vectorized instructions)

Cycles and memory resources usage

Assuming all data fit into the L1 cache, each iteration of the binary loop takes 22.00 cycles. At this rate:

12% of peak load performance is reached (11.64 out of 96.00 bytes loaded per cycle (GB/s @ 1GHz))
2% of peak store performance is reached (1.45 out of 64.00 bytes stored per cycle (GB/s @ 1GHz))

Front-end bottlenecks

Found no such bottlenecks.

ASM code

In the binary file, the address of the loop is: 234294

Instruction	Nb FU	ALU0/BRU0	ALU1	ALU2	ALU3	BRU1	AGU0	AGU1	AGU2	FP0	FP1	FP2	FP3	FP4	FP5	Latency	Recip. throughput
ADD %R10D,%EDI	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD %R10D,%EAX	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD %R11D,%R8D	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD %R11D,%EDX	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOVSXD %R8D,%R8	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOVSXD %EDX,%RDX	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOVSXD %EDI,%R13	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOVSXD %EAX,%R9	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV -0x58(%RBP),%RDI	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV %RDI,%RAX	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
IMUL %R13,%RAX	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
MOV -0xa0(%RBP),%R10	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV %R10,%R11	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
IMUL %R13,%R11	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
IMUL %R9,%R10	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
ADD %R8,%R11	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD %RDX,%R10	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV -0x98(%RBP),%R15	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVSD (%R15,%R11,8),%XMM0	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
VMOVHPD (%R15,%R10,8),%XMM0,%XMM0	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
IMUL %R9,%RDI	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
MOV -0x90(%RBP),%R11	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV %R11,%R10	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
IMUL %R13,%R10	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
IMUL %R9,%R11	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
LEA (%R10,%R8,1),%R15	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV -0x88(%RBP),%R14	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVSD (%R14,%R15,8),%XMM1	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
LEA (%R11,%RDX,1),%R15	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMOVHPD (%R14,%R15,8),%XMM1,%XMM1	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
LEA 0x1(%R10,%R8,1),%R10	2	0	0	0.50	0.50	0	0.33	0.33	0.33	0	0	0	0	0	0	2	0.50
LEA 0x1(%R11,%RDX,1),%R11	2	0	0	0.50	0.50	0	0.33	0.33	0.33	0	0	0	0	0	0	2	0.50
VMOVSD (%R14,%R10,8),%XMM2	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
VMOVHPD (%R14,%R11,8),%XMM2,%XMM2	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
MOV -0x78(%RBP),%R15	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV %R15,%R10	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
IMUL %R13,%R10	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
IMUL %R9,%R15	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
LEA (%R10,%R8,1),%R11	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV -0x70(%RBP),%RBX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVSD (%RBX,%R11,8),%XMM3	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
LEA (%R15,%RDX,1),%R11	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMOVHPD (%RBX,%R11,8),%XMM3,%XMM3	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
MOV -0xc8(%RBP),%R14	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV %R14,%R11	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
IMUL %R13,%R11	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
ADD %R8,%R10	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
INC %R10	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMOVSD (%RBX,%R10,8),%XMM4	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
MOV %R14,%R10	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
ADD %RDX,%R15	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
INC %R15	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMOVHPD (%RBX,%R15,8),%XMM4,%XMM4	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
IMUL %R9,%R10	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
MOV -0x68(%RBP),%RBX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
IMUL %RBX,%R13	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
IMUL %RBX,%R9	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
LEA (%R8,%R13,1),%R15	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV -0x60(%RBP),%RBX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVSD (%RBX,%R15,8),%XMM5	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
LEA (%R9,%RDX,1),%R15	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMOVHPD (%RBX,%R15,8),%XMM5,%XMM5	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
ADD %R8,%RAX	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD %R8,%R11	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD %R13,%R8	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
INC %R8	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMOVSD (%RBX,%R8,8),%XMM6	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
LEA (%R9,%RDX,1),%R8	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
INC %R8	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMOVHPD (%RBX,%R8,8),%XMM6,%XMM6	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
ADD %RDX,%RDI	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV -0x50(%RBP),%R8	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVSD (%R8,%RAX,8),%XMM7	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
VMOVHPD (%R8,%RDI,8),%XMM7,%XMM7	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
ADD %RDX,%R10	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMULPD %XMM0,%XMM7,%XMM7	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	3	0.50
VADDPD %XMM1,%XMM7,%XMM1	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
VSUBPD %XMM2,%XMM1,%XMM1	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
VADDPD %XMM5,%XMM0,%XMM0	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
MOV -0x80(%RBP),%RDX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVSD (%RDX,%R11,8),%XMM2	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
VMOVHPD (%RDX,%R10,8),%XMM2,%XMM2	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
VSUBPD %XMM6,%XMM0,%XMM0	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
VDIVPD %XMM0,%XMM1,%XMM0	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	13	5
VMOVLPD %XMM0,(%R8,%RAX,8)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0.50	0.50	1	1
VMOVHPD %XMM0,(%R8,%RDI,8)	2	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0.50	0.50	9-11	1
VMULPD %XMM7,%XMM2,%XMM0	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	3	0.50
VADDPD %XMM3,%XMM0,%XMM0	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
VSUBPD %XMM4,%XMM0,%XMM0	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
VDIVPD %XMM1,%XMM0,%XMM0	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	13	5
VMOVLPD %XMM0,(%RDX,%R11,8)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0.50	0.50	1	1
VMOVHPD %XMM0,(%RDX,%R10,8)	2	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0.50	0.50	9-11	1
ADD $0x2,%RSI	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD $-0x2,%R12	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
JE 2342ee <.omp_outlined..6+0x38e>	1	0.50	0	0	0	0.50	0	0	0	0	0	0	0	0	0	1	0.50-1
LEA 0x1(%RSI),%R9	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV %RSI,%RAX	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
SHR $0x20,%RAX	1	0	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	1	0.50
MOV -0x38(%RBP),%R15	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
JE 2342c0 <.omp_outlined..6+0x360>	1	0.50	0	0	0	0.50	0	0	0	0	0	0	0	0	0	1	0.50-1
MOV %ESI,%EAX	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
XOR %EDX,%EDX	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.25
DIV %R15D	2	2	0	0	0	0	0	0	0	0	0	0	0	0	0	10-17	6
MOV %EDX,%R8D	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
MOV %EAX,%EDI	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
MOV -0x30(%RBP),%R10D	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV -0x2c(%RBP),%R11D	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV %R9,%RAX	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
SHR $0x20,%RAX	1	0	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	1	0.50
JNE 2340f0 <.omp_outlined..6+0x190>	1	0.50	0	0	0	0.50	0	0	0	0	0	0	0	0	0	1	0.50-1
MOV %R9D,%EAX	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
XOR %EDX,%EDX	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.25
DIV %R15D	2	2	0	0	0	0	0	0	0	0	0	0	0	0	0	10-17	6
JMP 2340f8 <.omp_outlined..6+0x198>	1	0.50	0	0	0	0.50	0	0	0	0	0	0	0	0	0	0	1

Function	.omp_outlined..6
Source file and lines	advec_cell.cpp:118-125
Module	exec

The loop is defined in:

/beegfs/hackathon/users/eoseret/qaas_runs/170-854-8685/intel/CloverLeafCXX/build/CloverLeafCXX/src/omp/advec_cell.cpp:118-125
/beegfs/hackathon/users/eoseret/qaas_runs/170-854-8685/intel/CloverLeafCXX/build/CloverLeafCXX/src/omp/context.h:69

hoisting them (moving them outside the loop)
turning them into conditional moves, MIN or MAX

Ex: if (x<0) x=0 => x = (x<0 ? 0 : x) (or MAX(0,x) after defining the corresponding macro)

gain
potential
hint
expert

Code clean check

Workaround

Try to reorganize arrays of structures to structures of arrays
Consider to permute loops (see vectorization gain report)

Vectorization

Details

28% of SSE/AVX instructions are used in vector version (process two or more data elements in vector registers):

0% of SSE/AVX loads are used in vector version.
0% of SSE/AVX stores are used in vector version.
50% of SSE/AVX divide and square root instructions are used in vector version.
0% of SSE/AVX instructions that are not load, store, addition, subtraction nor multiply instructions are used in vector version.

Since your execution units are vector units, only a fully vectorized loop can use their full power.

Workaround

Try another compiler or update/tune your current one:
- recompile with ffast-math (included in Ofast) to extend loop vectorization to FP reductions.
Remove inter-iterations dependences from your loop and make it unit-stride:
- If your arrays have 2 or more dimensions, check whether elements are accessed contiguously and, otherwise, try to permute loops accordingly: C storage order is row-major: for(i) for(j) a[j][i] = b[j][i]; (slow, non stride 1) => for(i) for(j) a[i][j] = b[i][j]; (fast, stride 1)
- If your loop streams arrays of structures (AoS), try to use structures of arrays instead (SoA): for(i) a[i].x = b[i].x; (slow, non stride 1) => for(i) a.x[i] = b.x[i]; (fast, stride 1)

Execution units bottlenecks

Workaround

Reduce the number of division or square root instructions:
- If denominator is constant over iterations, use reciprocal (replace x/y with x*(1/y)). Check precision impact. This will be done by your compiler with ffast-math or Ofast
Check whether you really need double precision. If not, switch to single precision to speedup execution

Expensive FP math instructions/calls

Detected performance impact from expensive FP math instructions/calls. By removing/reexpressing them, you can lower the cost of an iteration from 28.00 to 19.33 cycles (1.45x speedup).

FMA

Presence of both ADD/SUB and MUL operations.

Workaround

Pass to your compiler a micro-architecture specialization option:
- use march=native
Try to change order in which elements are evaluated (using parentheses) in arithmetic expressions containing both ADD/SUB and MUL operations to enable your compiler to generate FMA instructions wherever possible. For instance a + b*c is a valid FMA (MUL then ADD). However (a+b)* c cannot be translated into an FMA (ADD then MUL).

Complex instructions

Detected COMPLEX INSTRUCTIONS.

Details

These instructions generate more than one micro-operation and only one of them can be decoded during a cycle and the extra micro-operations increase pressure on execution units.

DIV: 1 occurrences
IDIV: 1 occurrences
LEA: 2 occurrences
VMOVHPD: 2 occurrences

Slow data structures access

Detected data structures (typically arrays) that cannot be efficiently read/written

Details

Constant unknown stride: 13 occurrence(s)

Non-unit stride (uncontiguous) accesses are not efficiently using data caches

Workaround

Try to reorganize arrays of structures to structures of arrays
Consider to permute loops (see vectorization gain report)

Conversion instructions

Detected expensive conversion instructions.

Details

CQTO: 1 occurrences

Workaround

Avoid mixing data with different types. In particular, check if the type of constants is the same as array elements.

Type of elements and instruction set

10 SSE or AVX instructions are processing arithmetic or math operations on double precision FP elements in vector mode (two at a time).

Matching between your loop (in the source code) and the binary loop

The binary loop is composed of 20 FP arithmetical operations:

12: addition or subtraction
4: multiply
4: divide

The binary loop is loading 256 bytes (32 double precision FP elements). The binary loop is storing 32 bytes (4 double precision FP elements).

Arithmetic intensity

Arithmetic intensity is 0.07 FP operations per loaded or stored byte.

General properties

nb instructions	113
nb uops	118
loop length	470
used x86 registers	14
used mmx registers	0
used xmm registers	8
used ymm registers	0
used zmm registers	0
nb stack references	15
ADD-SUB / MUL ratio	3.00

Front-end

MACRO FUSION NOT POSSIBLE FIT IN UOP CACHE

micro-operation queue	19.67 cycles
front end	19.67 cycles

Back-end

	ALU0/BRU0	ALU1	ALU2	ALU3	BRU1	AGU0	AGU1	AGU2	FP0	FP1	FP2	FP3	FP4	FP5
uops	12.75	12.75	12.75	12.75	3.00	13.00	13.00	13.00	2.00	6.33	6.33	6.33	2.00	2.00
cycles	12.75	12.75	12.75	12.75	3.00	13.00	13.00	13.00	2.00	6.33	6.33	6.33	2.00	2.00

Execution ports to units layout:

ALU0/BRU0: ALU
ALU1: ALU
ALU2: ALU
ALU3: ALU
BRU1:
AGU0 (256 bits): store address, load
AGU1 (256 bits): store address, load
AGU2 (256 bits): store address, load
FP0 (256 bits): VPU, DIV/SQRT
FP1 (256 bits): VPU, DIV/SQRT
FP2 (256 bits): VPU
FP3 (256 bits): VPU
FP4 (256 bits): FP store data
FP5 (256 bits): FP store data

Cycles executing div or sqrt instructions	23.00-28.00
Longest recurrence chain latency (RecMII)	1.00

Cycles summary

Front-end	19.67
Dispatch	13.00
DIV/SQRT	23.00-28.00
Data deps.	1.00
Overall L1	23.00-28.00

Vectorization ratios

INT

all	0%
load	NA (no load vectorizable/vectorized instructions)
store	NA (no store vectorizable/vectorized instructions)
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	NA (no add-sub vectorizable/vectorized instructions)
fma	NA (no fma vectorizable/vectorized instructions)
other	0%

all	31%
load	0%
store	0%
mul	100%
add-sub	100%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	100%
other	NA (no other vectorizable/vectorized instructions)

INT+FP

all	28%
load	0%
store	0%
mul	100%
add-sub	100%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	50%
other	0%

Vector efficiency ratios

INT

all	10%
load	NA (no load vectorizable/vectorized instructions)
store	NA (no store vectorizable/vectorized instructions)
mul	NA (no mul vectorizable/vectorized instructions)
add-sub	NA (no add-sub vectorizable/vectorized instructions)
fma	NA (no fma vectorizable/vectorized instructions)
other	10%

all	16%
load	12%
store	12%
mul	25%
add-sub	25%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	25%
other	NA (no other vectorizable/vectorized instructions)

INT+FP

all	15%
load	12%
store	12%
mul	25%
add-sub	25%
fma	NA (no fma vectorizable/vectorized instructions)
div/sqrt	17%
other	12%

Cycles and memory resources usage

Assuming all data fit into the L1 cache, each iteration of the binary loop takes 28.00 cycles. At this rate:

9% of peak load performance is reached (9.14 out of 96.00 bytes loaded per cycle (GB/s @ 1GHz))
1% of peak store performance is reached (1.14 out of 64.00 bytes stored per cycle (GB/s @ 1GHz))

Front-end bottlenecks

Found no such bottlenecks.

ASM code

In the binary file, the address of the loop is: 234294

Instruction	Nb FU	ALU0/BRU0	ALU1	ALU2	ALU3	BRU1	AGU0	AGU1	AGU2	FP0	FP1	FP2	FP3	FP4	FP5	Latency	Recip. throughput
MOV %R9,%RAX	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
CQTO
IDIV %R15	2	2	0	0	0	0	0	0	0	0	0	0	0	0	0	9-19	7-12
ADD %R10D,%EDI	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD %R10D,%EAX	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD %R11D,%R8D	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD %R11D,%EDX	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOVSXD %R8D,%R8	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOVSXD %EDX,%RDX	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOVSXD %EDI,%R13	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOVSXD %EAX,%R9	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV -0x58(%RBP),%RDI	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV %RDI,%RAX	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
IMUL %R13,%RAX	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
MOV -0xa0(%RBP),%R10	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV %R10,%R11	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
IMUL %R13,%R11	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
IMUL %R9,%R10	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
ADD %R8,%R11	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD %RDX,%R10	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV -0x98(%RBP),%R15	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVSD (%R15,%R11,8),%XMM0	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
VMOVHPD (%R15,%R10,8),%XMM0,%XMM0	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
IMUL %R9,%RDI	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
MOV -0x90(%RBP),%R11	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV %R11,%R10	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
IMUL %R13,%R10	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
IMUL %R9,%R11	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
LEA (%R10,%R8,1),%R15	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV -0x88(%RBP),%R14	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVSD (%R14,%R15,8),%XMM1	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
LEA (%R11,%RDX,1),%R15	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMOVHPD (%R14,%R15,8),%XMM1,%XMM1	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
LEA 0x1(%R10,%R8,1),%R10	2	0	0	0.50	0.50	0	0.33	0.33	0.33	0	0	0	0	0	0	2	0.50
LEA 0x1(%R11,%RDX,1),%R11	2	0	0	0.50	0.50	0	0.33	0.33	0.33	0	0	0	0	0	0	2	0.50
VMOVSD (%R14,%R10,8),%XMM2	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
VMOVHPD (%R14,%R11,8),%XMM2,%XMM2	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
MOV -0x78(%RBP),%R15	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV %R15,%R10	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
IMUL %R13,%R10	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
IMUL %R9,%R15	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
LEA (%R10,%R8,1),%R11	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV -0x70(%RBP),%RBX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVSD (%RBX,%R11,8),%XMM3	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
LEA (%R15,%RDX,1),%R11	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMOVHPD (%RBX,%R11,8),%XMM3,%XMM3	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
MOV -0xc8(%RBP),%R14	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV %R14,%R11	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
IMUL %R13,%R11	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
ADD %R8,%R10	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
INC %R10	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMOVSD (%RBX,%R10,8),%XMM4	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
MOV %R14,%R10	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
ADD %RDX,%R15	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
INC %R15	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMOVHPD (%RBX,%R15,8),%XMM4,%XMM4	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
IMUL %R9,%R10	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
MOV -0x68(%RBP),%RBX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
IMUL %RBX,%R13	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
IMUL %RBX,%R9	1	0	1	0	0	0	0	0	0	0	0	0	0	0	0	3	1
LEA (%R8,%R13,1),%R15	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV -0x60(%RBP),%RBX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVSD (%RBX,%R15,8),%XMM5	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
LEA (%R9,%RDX,1),%R15	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMOVHPD (%RBX,%R15,8),%XMM5,%XMM5	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
ADD %R8,%RAX	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD %R8,%R11	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD %R13,%R8	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
INC %R8	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMOVSD (%RBX,%R8,8),%XMM6	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
LEA (%R9,%RDX,1),%R8	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
INC %R8	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMOVHPD (%RBX,%R8,8),%XMM6,%XMM6	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
ADD %RDX,%RDI	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV -0x50(%RBP),%R8	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVSD (%R8,%RAX,8),%XMM7	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
VMOVHPD (%R8,%RDI,8),%XMM7,%XMM7	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
ADD %RDX,%R10	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
VMULPD %XMM0,%XMM7,%XMM7	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	3	0.50
VADDPD %XMM1,%XMM7,%XMM1	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
VSUBPD %XMM2,%XMM1,%XMM1	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
VADDPD %XMM5,%XMM0,%XMM0	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
MOV -0x80(%RBP),%RDX	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
VMOVSD (%RDX,%R11,8),%XMM2	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	1	0.50
VMOVHPD (%RDX,%R10,8),%XMM2,%XMM2	1	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0	0	9	0.50
VSUBPD %XMM6,%XMM0,%XMM0	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
VDIVPD %XMM0,%XMM1,%XMM0	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	13	5
VMOVLPD %XMM0,(%R8,%RAX,8)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0.50	0.50	1	1
VMOVHPD %XMM0,(%R8,%RDI,8)	2	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0.50	0.50	9-11	1
VMULPD %XMM7,%XMM2,%XMM0	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	3	0.50
VADDPD %XMM3,%XMM0,%XMM0	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
VSUBPD %XMM4,%XMM0,%XMM0	1	0	0	0	0	0	0	0	0	0	0	0.50	0.50	0	0	3	0.50
VDIVPD %XMM1,%XMM0,%XMM0	1	0	0	0	0	0	0	0	0	0.50	0.50	0	0	0	0	13	5
VMOVLPD %XMM0,(%RDX,%R11,8)	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0.50	0.50	1	1
VMOVHPD %XMM0,(%RDX,%R10,8)	2	0	0	0	0	0	0.33	0.33	0.33	0	0.33	0.33	0.33	0.50	0.50	9-11	1
ADD $0x2,%RSI	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
ADD $-0x2,%R12	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
JE 2342ee <.omp_outlined..6+0x38e>	1	0.50	0	0	0	0.50	0	0	0	0	0	0	0	0	0	1	0.50-1
LEA 0x1(%RSI),%R9	1	0.25	0.25	0.25	0.25	0	0	0	0	0	0	0	0	0	0	1	0.25
MOV %RSI,%RAX	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
SHR $0x20,%RAX	1	0	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	1	0.50
MOV -0x38(%RBP),%R15	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
JE 2342c0 <.omp_outlined..6+0x360>	1	0.50	0	0	0	0.50	0	0	0	0	0	0	0	0	0	1	0.50-1
MOV %ESI,%EAX	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
XOR %EDX,%EDX	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.25
DIV %R15D	2	2	0	0	0	0	0	0	0	0	0	0	0	0	0	10-17	6
MOV %EDX,%R8D	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
MOV %EAX,%EDI	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
MOV -0x30(%RBP),%R10D	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV -0x2c(%RBP),%R11D	1	0	0	0	0	0	0.33	0.33	0.33	0	0	0	0	0	0	3	0.33
MOV %R9,%RAX	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0.17
SHR $0x20,%RAX	1	0	0.50	0.50	0	0	0	0	0	0	0	0	0	0	0	1	0.50
JNE 2340f0 <.omp_outlined..6+0x190>	1	0.50	0	0	0	0.50	0	0	0	0	0	0	0	0	0	1	0.50-1

▶Scalability Runs Description

Run 2x1	Number processes: 2Number nodes: 1Number processes per node: 2Run Command: <executable> MPI Command: mpirun -np <number_processes>Dataset: Run Directory: /beegfs/hackathon/users/eoseret/qaas_runs/170-854-8685/intel/CloverLeafCXX/run/oneview_runs/compilers/aocc_15/oneview_run_1708555871I_MPI_PIN_DOMAIN: auto:scatterOMP_PLACES: threadsOMP_PROC_BIND: spreadOMP_NUM_THREADS: 1
Run 2x2	OMP_NUM_THREADS: 2I_MPI_PIN_DOMAIN: auto:scatterOMP_PLACES: threadsOMP_PROC_BIND: spread
Run 2x4	OMP_NUM_THREADS: 4I_MPI_PIN_DOMAIN: auto:scatterOMP_PLACES: threadsOMP_PROC_BIND: spread
Run 2x8	OMP_NUM_THREADS: 8I_MPI_PIN_DOMAIN: auto:scatterOMP_PLACES: threadsOMP_PROC_BIND: spread
Run 2x16	OMP_NUM_THREADS: 16I_MPI_PIN_DOMAIN: auto:scatterOMP_PLACES: threadsOMP_PROC_BIND: spread
Run 2x32	OMP_NUM_THREADS: 32I_MPI_PIN_DOMAIN: auto:scatterOMP_PLACES: threadsOMP_PROC_BIND: spread
Run 2x64	OMP_NUM_THREADS: 64I_MPI_PIN_DOMAIN: auto:scatterOMP_PLACES: threadsOMP_PROC_BIND: spread
Run 2x96	OMP_NUM_THREADS: 96I_MPI_PIN_DOMAIN: auto:scatterOMP_PLACES: threadsOMP_PROC_BIND: spread

(2x1) Efficiency	(2x1) Potential Speed-Up (%)	(2x2) Efficiency	(2x2) Potential Speed-Up (%)	(2x4) Efficiency	(2x4) Potential Speed-Up (%)	(2x8) Efficiency	(2x8) Potential Speed-Up (%)	(2x16) Efficiency	(2x16) Potential Speed-Up (%)	(2x32) Efficiency	(2x32) Potential Speed-Up (%)	(2x64) Efficiency	(2x64) Potential Speed-Up (%)	(2x96) Efficiency	(2x96) Potential Speed-Up (%)
1	0	1	0	1	0	1	0	0.98	0.04	0.73	0.73	0.37	1.88	0.25	2.52

Speed-Up and Efficiency

▶Detailed Speed-Up and Efficiency

Run	Number of threads	Efficiency (ideal is 1)	Speedup	Ideal Speedup	Time (s)	Coverage (%)
2x1	2	1	1	1	41.78	2.43
2x2	4	1	2	2	20.95	2.43
2x4	8	1	3.99	4	10.48	2.42
2x8	16	1	7.99	8	5.25	2.42
2x16	32	0.98	15.74	16	2.74	2.32
2x32	64	0.73	23.31	32	2.01	2.68
2x64	128	0.37	23.57	64	2.03	2.98
2x96	192	0.25	23.7	96	1.81	3.35

Report Configuration

Vectorization

Details

Workaround

Expensive FP math instructions/calls

FMA

Workaround

Complex instructions

Details

Conversion instructions

Details

Workaround

Type of elements and instruction set

Matching between your loop (in the source code) and the binary loop

Arithmetic intensity

General properties

Front-end

Back-end

Cycles summary

Vectorization ratios

Vector efficiency ratios

Cycles and memory resources usage

Code clean check

Workaround

Vectorization

Details

Workaround

Execution units bottlenecks

Workaround

Expensive FP math instructions/calls

FMA

Workaround

Complex instructions

Details

Slow data structures access

Details

Workaround

Conversion instructions

Details

Workaround

Type of elements and instruction set

Matching between your loop (in the source code) and the binary loop

Arithmetic intensity

General properties

Front-end

Back-end

Cycles summary

Vectorization ratios

Vector efficiency ratios

Cycles and memory resources usage

Front-end bottlenecks

ASM code

Code clean check

Workaround

Vectorization

Details

Workaround

Execution units bottlenecks

Workaround

Expensive FP math instructions/calls

FMA

Workaround

Complex instructions

Details

Slow data structures access

Details

Workaround

Conversion instructions

Details

Workaround

Type of elements and instruction set

Matching between your loop (in the source code) and the binary loop

Arithmetic intensity

General properties

Front-end

Back-end

Cycles summary

Vectorization ratios

Vector efficiency ratios

Cycles and memory resources usage